chardet 0.9.0

data/lib/Latin1Prober.rb

@@ -0,0 +1,160 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetProber'
+
+module Enumerable
+    def reduce(fn, res)
+        each { |n| res = res.send(fn, n) }
+        res
+    end
+
+    def reduceBlock(res)
+        each { |n| res = yield(res, n) }
+        res
+    end
+end
+
+module UniversalDetector
+    FREQ_CAT_NUM = 4
+
+    UDF = 0 # undefined
+    OTH = 1 # other
+    ASC = 2 # ascii capital letter
+    ASS = 3 # ascii small letter
+    ACV = 4 # accent capital vowel
+    ACO = 5 # accent capital other
+    ASV = 6 # accent small vowel
+    ASO = 7 # accent small other
+    CLASS_NUM = 8 # total classes
+
+    Latin1_CharToClass = [ \
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 00 - 07
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 08 - 0F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 10 - 17
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 18 - 1F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 20 - 27
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 28 - 2F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 30 - 37
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 38 - 3F
+      OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 40 - 47
+      ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 48 - 4F
+      ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 50 - 57
+      ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   # 58 - 5F
+      OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 60 - 67
+      ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 68 - 6F
+      ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 70 - 77
+      ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   # 78 - 7F
+      OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   # 80 - 87
+      OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   # 88 - 8F
+      UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 90 - 97
+      OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   # 98 - 9F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A0 - A7
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A8 - AF
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B0 - B7
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B8 - BF
+      ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   # C0 - C7
+      ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   # C8 - CF
+      ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   # D0 - D7
+      ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   # D8 - DF
+      ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   # E0 - E7
+      ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   # E8 - EF
+      ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   # F0 - F7
+      ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO,   # F8 - FF
+    ]
+
+    # 0 : illegal 
+    # 1 : very unlikely 
+    # 2 : normal 
+    # 3 : very likely
+    Latin1ClassModel = [ \
+    # UDF OTH ASC ASS ACV ACO ASV ASO
+       0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+       0,  3,  3,  3,  3,  3,  3,  3,  # OTH
+       0,  3,  3,  3,  3,  3,  3,  3,  # ASC
+       0,  3,  3,  3,  1,  1,  3,  3,  # ASS
+       0,  3,  3,  3,  1,  2,  1,  2,  # ACV
+       0,  3,  3,  3,  3,  3,  3,  3,  # ACO
+       0,  3,  1,  3,  1,  1,  1,  3,  # ASV
+       0,  3,  1,  3,  1,  1,  3,  3,  # ASO
+    ]
+
+    class Latin1Prober < CharSetProber
+        def initialize
+            super
+            reset()
+        end
+
+        def reset
+            @_mLastCharClass = OTH
+            @_mFreqCounter = [0] * FREQ_CAT_NUM
+            super
+        end
+
+        def get_charset_name
+            return "windows-1252"
+        end
+
+        def feed(aBuf)
+            aBuf = filter_with_english_letters(aBuf)
+            for c in aBuf
+                charClass = Latin1_CharToClass[c[0]]
+                freq = Latin1ClassModel[(@_mLastCharClass * CLASS_NUM) + charClass]
+                if freq == 0
+                    @_mState = :NotMe
+                    break
+                end
+                @_mFreqCounter[freq] += 1
+                @_mLastCharClass = charClass
+            end        
+
+            return get_state()
+        end
+
+        def get_confidence()
+            if get_state() == :NotMe
+                return 0.01
+            end
+
+            total = @_mFreqCounter.reduce(:+, 0)
+            if total < 0.01
+                confidence = 0.0
+            else
+                confidence = (@_mFreqCounter[3] / total) - (@_mFreqCounter[1] * 20.0 / total)
+            end
+            if confidence < 0.0
+                confidence = 0.0
+            end
+            # lower the confidence of latin1 so that other more accurate detector 
+            # can take priority.
+            confidence = confidence * 0.5
+            return confidence
+        end
+    end
+end

data/lib/MBCSGroupProber.rb

@@ -0,0 +1,57 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetGroupProber'
+require 'UTF8Prober'
+require 'SJISProber'
+require 'EUCJPProber'
+require 'GB2312Prober'
+require 'EUCKRProber'
+require 'Big5Prober'
+require 'EUCTWProber'
+
+module UniversalDetector
+    class MBCSGroupProber < CharSetGroupProber
+        
+        attr_reader :mProbers
+        
+        def initialize
+            super
+            @mProbers = [ \
+                UTF8Prober.new,
+                SJISProber.new,
+                EUCJPProber.new,
+                GB2312Prober.new,
+                EUCKRProber.new,
+                Big5Prober.new,
+                EUCTWProber.new]
+            reset()
+        end
+    end
+end

data/lib/MBCSSM.rb

@@ -0,0 +1,513 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module UniversalDetector
+    BIG5_cls = [ \
+        1,1,1,1,1,1,1,1,  # 00 - 07    #allow 0x00 as legal value
+        1,1,1,1,1,1,0,0,  # 08 - 0f 
+        1,1,1,1,1,1,1,1,  # 10 - 17 
+        1,1,1,0,1,1,1,1,  # 18 - 1f 
+        1,1,1,1,1,1,1,1,  # 20 - 27 
+        1,1,1,1,1,1,1,1,  # 28 - 2f 
+        1,1,1,1,1,1,1,1,  # 30 - 37 
+        1,1,1,1,1,1,1,1,  # 38 - 3f 
+        2,2,2,2,2,2,2,2,  # 40 - 47 
+        2,2,2,2,2,2,2,2,  # 48 - 4f 
+        2,2,2,2,2,2,2,2,  # 50 - 57 
+        2,2,2,2,2,2,2,2,  # 58 - 5f 
+        2,2,2,2,2,2,2,2,  # 60 - 67 
+        2,2,2,2,2,2,2,2,  # 68 - 6f 
+        2,2,2,2,2,2,2,2,  # 70 - 77 
+        2,2,2,2,2,2,2,1,  # 78 - 7f 
+        4,4,4,4,4,4,4,4,  # 80 - 87 
+        4,4,4,4,4,4,4,4,  # 88 - 8f 
+        4,4,4,4,4,4,4,4,  # 90 - 97 
+        4,4,4,4,4,4,4,4,  # 98 - 9f 
+        4,3,3,3,3,3,3,3,  # a0 - a7 
+        3,3,3,3,3,3,3,3,  # a8 - af 
+        3,3,3,3,3,3,3,3,  # b0 - b7 
+        3,3,3,3,3,3,3,3,  # b8 - bf 
+        3,3,3,3,3,3,3,3,  # c0 - c7 
+        3,3,3,3,3,3,3,3,  # c8 - cf 
+        3,3,3,3,3,3,3,3,  # d0 - d7 
+        3,3,3,3,3,3,3,3,  # d8 - df 
+        3,3,3,3,3,3,3,3,  # e0 - e7 
+        3,3,3,3,3,3,3,3,  # e8 - ef 
+        3,3,3,3,3,3,3,3,  # f0 - f7 
+        3,3,3,3,3,3,3,0]  # f8 - ff 
+
+    BIG5_st = [ \
+        :Error,:Start,:Start,     3,:Error,:Error,:Error,:Error,#00-07 
+        :Error,:Error,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:Error,#08-0f 
+        :Error,:Start,:Start,:Start,:Start,:Start,:Start,:Start]#10-17 
+
+    Big5CharLenTable = [0, 1, 1, 2, 0]
+
+    Big5SMModel = {'classTable' =>  BIG5_cls,
+                   'classFactor' =>  5,
+                   'stateTable' =>  BIG5_st,
+                   'charLenTable' =>  Big5CharLenTable,
+                   'name' =>  'Big5'}
+
+    # EUC-JP
+
+    EUCJP_cls = [ \
+        4,4,4,4,4,4,4,4,  # 00 - 07 
+        4,4,4,4,4,4,5,5,  # 08 - 0f 
+        4,4,4,4,4,4,4,4,  # 10 - 17 
+        4,4,4,5,4,4,4,4,  # 18 - 1f 
+        4,4,4,4,4,4,4,4,  # 20 - 27 
+        4,4,4,4,4,4,4,4,  # 28 - 2f 
+        4,4,4,4,4,4,4,4,  # 30 - 37 
+        4,4,4,4,4,4,4,4,  # 38 - 3f 
+        4,4,4,4,4,4,4,4,  # 40 - 47 
+        4,4,4,4,4,4,4,4,  # 48 - 4f 
+        4,4,4,4,4,4,4,4,  # 50 - 57 
+        4,4,4,4,4,4,4,4,  # 58 - 5f 
+        4,4,4,4,4,4,4,4,  # 60 - 67 
+        4,4,4,4,4,4,4,4,  # 68 - 6f 
+        4,4,4,4,4,4,4,4,  # 70 - 77 
+        4,4,4,4,4,4,4,4,  # 78 - 7f 
+        5,5,5,5,5,5,5,5,  # 80 - 87 
+        5,5,5,5,5,5,1,3,  # 88 - 8f 
+        5,5,5,5,5,5,5,5,  # 90 - 97 
+        5,5,5,5,5,5,5,5,  # 98 - 9f 
+        5,2,2,2,2,2,2,2,  # a0 - a7 
+        2,2,2,2,2,2,2,2,  # a8 - af 
+        2,2,2,2,2,2,2,2,  # b0 - b7 
+        2,2,2,2,2,2,2,2,  # b8 - bf 
+        2,2,2,2,2,2,2,2,  # c0 - c7 
+        2,2,2,2,2,2,2,2,  # c8 - cf 
+        2,2,2,2,2,2,2,2,  # d0 - d7 
+        2,2,2,2,2,2,2,2,  # d8 - df 
+        0,0,0,0,0,0,0,0,  # e0 - e7 
+        0,0,0,0,0,0,0,0,  # e8 - ef 
+        0,0,0,0,0,0,0,0,  # f0 - f7 
+        0,0,0,0,0,0,0,5]  # f8 - ff 
+
+    EUCJP_st = [ \
+              3,     4,     3,     5,:Start,:Error,:Error,:Error,#00-07 
+         :Error,:Error,:Error,:Error,:ItsMe,:ItsMe,:ItsMe,:ItsMe,#08-0f 
+         :ItsMe,:ItsMe,:Start,:Error,:Start,:Error,:Error,:Error,#10-17 
+         :Error,:Error,:Start,:Error,:Error,:Error,     3,:Error,#18-1f 
+              3,:Error,:Error,:Error,:Start,:Start,:Start,:Start]#20-27 
+
+    EUCJPCharLenTable = [2, 2, 2, 3, 1, 0]
+
+    EUCJPSMModel = {'classTable' =>  EUCJP_cls,
+                    'classFactor' =>  6,
+                    'stateTable' =>  EUCJP_st,
+                    'charLenTable' =>  EUCJPCharLenTable,
+                    'name' =>  'EUC-JP'}
+
+    # EUC-KR
+
+    EUCKR_cls  = [ \
+        1,1,1,1,1,1,1,1,  # 00 - 07 
+        1,1,1,1,1,1,0,0,  # 08 - 0f 
+        1,1,1,1,1,1,1,1,  # 10 - 17 
+        1,1,1,0,1,1,1,1,  # 18 - 1f 
+        1,1,1,1,1,1,1,1,  # 20 - 27 
+        1,1,1,1,1,1,1,1,  # 28 - 2f 
+        1,1,1,1,1,1,1,1,  # 30 - 37 
+        1,1,1,1,1,1,1,1,  # 38 - 3f 
+        1,1,1,1,1,1,1,1,  # 40 - 47 
+        1,1,1,1,1,1,1,1,  # 48 - 4f 
+        1,1,1,1,1,1,1,1,  # 50 - 57 
+        1,1,1,1,1,1,1,1,  # 58 - 5f 
+        1,1,1,1,1,1,1,1,  # 60 - 67 
+        1,1,1,1,1,1,1,1,  # 68 - 6f 
+        1,1,1,1,1,1,1,1,  # 70 - 77 
+        1,1,1,1,1,1,1,1,  # 78 - 7f 
+        0,0,0,0,0,0,0,0,  # 80 - 87 
+        0,0,0,0,0,0,0,0,  # 88 - 8f 
+        0,0,0,0,0,0,0,0,  # 90 - 97 
+        0,0,0,0,0,0,0,0,  # 98 - 9f 
+        0,2,2,2,2,2,2,2,  # a0 - a7 
+        2,2,2,2,2,3,3,3,  # a8 - af 
+        2,2,2,2,2,2,2,2,  # b0 - b7 
+        2,2,2,2,2,2,2,2,  # b8 - bf 
+        2,2,2,2,2,2,2,2,  # c0 - c7 
+        2,3,2,2,2,2,2,2,  # c8 - cf 
+        2,2,2,2,2,2,2,2,  # d0 - d7 
+        2,2,2,2,2,2,2,2,  # d8 - df 
+        2,2,2,2,2,2,2,2,  # e0 - e7 
+        2,2,2,2,2,2,2,2,  # e8 - ef 
+        2,2,2,2,2,2,2,2,  # f0 - f7 
+        2,2,2,2,2,2,2,0]  # f8 - ff 
+
+    EUCKR_st = [
+        :Error,:Start,     3,:Error,:Error,:Error,:Error,:Error,#00-07 
+        :ItsMe,:ItsMe,:ItsMe,:ItsMe,:Error,:Error,:Start,:Start]#08-0f 
+
+    EUCKRCharLenTable = [0, 1, 2, 0]
+
+    EUCKRSMModel = {'classTable' =>  EUCKR_cls,
+                    'classFactor' =>  4,
+                    'stateTable' =>  EUCKR_st,
+                    'charLenTable' =>  EUCKRCharLenTable,
+                    'name' =>  'EUC-KR'}
+
+    # EUC-TW
+
+    EUCTW_cls = [ \
+        2,2,2,2,2,2,2,2,  # 00 - 07 
+        2,2,2,2,2,2,0,0,  # 08 - 0f 
+        2,2,2,2,2,2,2,2,  # 10 - 17 
+        2,2,2,0,2,2,2,2,  # 18 - 1f 
+        2,2,2,2,2,2,2,2,  # 20 - 27 
+        2,2,2,2,2,2,2,2,  # 28 - 2f 
+        2,2,2,2,2,2,2,2,  # 30 - 37 
+        2,2,2,2,2,2,2,2,  # 38 - 3f 
+        2,2,2,2,2,2,2,2,  # 40 - 47 
+        2,2,2,2,2,2,2,2,  # 48 - 4f 
+        2,2,2,2,2,2,2,2,  # 50 - 57 
+        2,2,2,2,2,2,2,2,  # 58 - 5f 
+        2,2,2,2,2,2,2,2,  # 60 - 67 
+        2,2,2,2,2,2,2,2,  # 68 - 6f 
+        2,2,2,2,2,2,2,2,  # 70 - 77 
+        2,2,2,2,2,2,2,2,  # 78 - 7f 
+        0,0,0,0,0,0,0,0,  # 80 - 87 
+        0,0,0,0,0,0,6,0,  # 88 - 8f 
+        0,0,0,0,0,0,0,0,  # 90 - 97 
+        0,0,0,0,0,0,0,0,  # 98 - 9f 
+        0,3,4,4,4,4,4,4,  # a0 - a7 
+        5,5,1,1,1,1,1,1,  # a8 - af 
+        1,1,1,1,1,1,1,1,  # b0 - b7 
+        1,1,1,1,1,1,1,1,  # b8 - bf 
+        1,1,3,1,3,3,3,3,  # c0 - c7 
+        3,3,3,3,3,3,3,3,  # c8 - cf 
+        3,3,3,3,3,3,3,3,  # d0 - d7 
+        3,3,3,3,3,3,3,3,  # d8 - df 
+        3,3,3,3,3,3,3,3,  # e0 - e7 
+        3,3,3,3,3,3,3,3,  # e8 - ef 
+        3,3,3,3,3,3,3,3,  # f0 - f7 
+        3,3,3,3,3,3,3,0]  # f8 - ff 
+
+    EUCTW_st = [ \
+        :Error,:Error,:Start,     3,     3,     3,     4,:Error,#00-07 
+        :Error,:Error,:Error,:Error,:Error,:Error,:ItsMe,:ItsMe,#08-0f 
+        :ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:Error,:Start,:Error,#10-17 
+        :Start,:Start,:Start,:Error,:Error,:Error,:Error,:Error,#18-1f 
+             5,:Error,:Error,:Error,:Start,:Error,:Start,:Start,#20-27 
+        :Start,:Error,:Start,:Start,:Start,:Start,:Start,:Start]#28-2f 
+
+    EUCTWCharLenTable = [0, 0, 1, 2, 2, 2, 3]
+
+    EUCTWSMModel = {'classTable' =>  EUCTW_cls,
+                    'classFactor' =>  7,
+                    'stateTable' =>  EUCTW_st,
+                    'charLenTable' =>  EUCTWCharLenTable,
+                    'name' =>  'x-euc-tw'}
+
+    # GB2312
+
+    GB2312_cls = [ \
+        1,1,1,1,1,1,1,1,  # 00 - 07 
+        1,1,1,1,1,1,0,0,  # 08 - 0f 
+        1,1,1,1,1,1,1,1,  # 10 - 17 
+        1,1,1,0,1,1,1,1,  # 18 - 1f 
+        1,1,1,1,1,1,1,1,  # 20 - 27 
+        1,1,1,1,1,1,1,1,  # 28 - 2f 
+        3,3,3,3,3,3,3,3,  # 30 - 37 
+        3,3,1,1,1,1,1,1,  # 38 - 3f 
+        2,2,2,2,2,2,2,2,  # 40 - 47 
+        2,2,2,2,2,2,2,2,  # 48 - 4f 
+        2,2,2,2,2,2,2,2,  # 50 - 57 
+        2,2,2,2,2,2,2,2,  # 58 - 5f 
+        2,2,2,2,2,2,2,2,  # 60 - 67 
+        2,2,2,2,2,2,2,2,  # 68 - 6f 
+        2,2,2,2,2,2,2,2,  # 70 - 77 
+        2,2,2,2,2,2,2,4,  # 78 - 7f 
+        5,6,6,6,6,6,6,6,  # 80 - 87 
+        6,6,6,6,6,6,6,6,  # 88 - 8f 
+        6,6,6,6,6,6,6,6,  # 90 - 97 
+        6,6,6,6,6,6,6,6,  # 98 - 9f 
+        6,6,6,6,6,6,6,6,  # a0 - a7 
+        6,6,6,6,6,6,6,6,  # a8 - af 
+        6,6,6,6,6,6,6,6,  # b0 - b7 
+        6,6,6,6,6,6,6,6,  # b8 - bf 
+        6,6,6,6,6,6,6,6,  # c0 - c7 
+        6,6,6,6,6,6,6,6,  # c8 - cf 
+        6,6,6,6,6,6,6,6,  # d0 - d7 
+        6,6,6,6,6,6,6,6,  # d8 - df 
+        6,6,6,6,6,6,6,6,  # e0 - e7 
+        6,6,6,6,6,6,6,6,  # e8 - ef 
+        6,6,6,6,6,6,6,6,  # f0 - f7 
+        6,6,6,6,6,6,6,0]  # f8 - ff 
+
+    GB2312_st = [ \
+        :Error,:Start,:Start,:Start,:Start,:Start,     3,:Error,#00-07 
+        :Error,:Error,:Error,:Error,:Error,:Error,:ItsMe,:ItsMe,#08-0f 
+        :ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:Error,:Error,:Start,#10-17 
+             4,:Error,:Start,:Start,:Error,:Error,:Error,:Error,#18-1f 
+        :Error,:Error,     5,:Error,:Error,:Error,:ItsMe,:Error,#20-27 
+        :Error,:Error,:Start,:Start,:Start,:Start,:Start,:Start]#28-2f 
+
+    # To be accurate, the length of class 6 can be either 2 or 4. 
+    # But it is not necessary to discriminate between the two since 
+    # it is used for frequency analysis only, and we are validing 
+    # each code range there as well. So it is safe to set it to be 
+    # 2 here. 
+    GB2312CharLenTable = [0, 1, 1, 1, 1, 1, 2]
+
+    GB2312SMModel = {'classTable' =>  GB2312_cls,
+                      'classFactor' =>  7,
+                      'stateTable' =>  GB2312_st,
+                      'charLenTable' =>  GB2312CharLenTable,
+                      'name' =>  'GB2312'}
+
+    # Shift_JIS
+
+    SJIS_cls = [ \
+        1,1,1,1,1,1,1,1,  # 00 - 07 
+        1,1,1,1,1,1,0,0,  # 08 - 0f 
+        1,1,1,1,1,1,1,1,  # 10 - 17 
+        1,1,1,0,1,1,1,1,  # 18 - 1f 
+        1,1,1,1,1,1,1,1,  # 20 - 27 
+        1,1,1,1,1,1,1,1,  # 28 - 2f 
+        1,1,1,1,1,1,1,1,  # 30 - 37 
+        1,1,1,1,1,1,1,1,  # 38 - 3f 
+        2,2,2,2,2,2,2,2,  # 40 - 47 
+        2,2,2,2,2,2,2,2,  # 48 - 4f 
+        2,2,2,2,2,2,2,2,  # 50 - 57 
+        2,2,2,2,2,2,2,2,  # 58 - 5f 
+        2,2,2,2,2,2,2,2,  # 60 - 67 
+        2,2,2,2,2,2,2,2,  # 68 - 6f 
+        2,2,2,2,2,2,2,2,  # 70 - 77 
+        2,2,2,2,2,2,2,1,  # 78 - 7f 
+        3,3,3,3,3,3,3,3,  # 80 - 87 
+        3,3,3,3,3,3,3,3,  # 88 - 8f 
+        3,3,3,3,3,3,3,3,  # 90 - 97 
+        3,3,3,3,3,3,3,3,  # 98 - 9f 
+        #0xa0 is illegal in sjis encoding, but some pages does 
+        #contain such byte. We need to be more error forgiven.
+        2,2,2,2,2,2,2,2,  # a0 - a7     
+        2,2,2,2,2,2,2,2,  # a8 - af 
+        2,2,2,2,2,2,2,2,  # b0 - b7 
+        2,2,2,2,2,2,2,2,  # b8 - bf 
+        2,2,2,2,2,2,2,2,  # c0 - c7 
+        2,2,2,2,2,2,2,2,  # c8 - cf 
+        2,2,2,2,2,2,2,2,  # d0 - d7 
+        2,2,2,2,2,2,2,2,  # d8 - df 
+        3,3,3,3,3,3,3,3,  # e0 - e7 
+        3,3,3,3,3,4,4,4,  # e8 - ef 
+        4,4,4,4,4,4,4,4,  # f0 - f7 
+        4,4,4,4,4,0,0,0]  # f8 - ff 
+
+    SJIS_st = [ \
+        :Error,:Start,:Start,     3,:Error,:Error,:Error,:Error,#00-07 
+        :Error,:Error,:Error,:Error,:ItsMe,:ItsMe,:ItsMe,:ItsMe,#08-0f 
+        :ItsMe,:ItsMe,:Error,:Error,:Start,:Start,:Start,:Start]#10-17 
+
+    SJISCharLenTable = [0, 1, 1, 2, 0, 0]
+
+    SJISSMModel = {'classTable' =>  SJIS_cls,
+                   'classFactor' =>  6,
+                   'stateTable' =>  SJIS_st,
+                   'charLenTable' =>  SJISCharLenTable,
+                   'name' =>  'Shift_JIS'}
+
+    # UCS2-BE
+
+    UCS2BE_cls = [ \
+        0,0,0,0,0,0,0,0,  # 00 - 07 
+        0,0,1,0,0,2,0,0,  # 08 - 0f 
+        0,0,0,0,0,0,0,0,  # 10 - 17 
+        0,0,0,3,0,0,0,0,  # 18 - 1f 
+        0,0,0,0,0,0,0,0,  # 20 - 27 
+        0,3,3,3,3,3,0,0,  # 28 - 2f 
+        0,0,0,0,0,0,0,0,  # 30 - 37 
+        0,0,0,0,0,0,0,0,  # 38 - 3f 
+        0,0,0,0,0,0,0,0,  # 40 - 47 
+        0,0,0,0,0,0,0,0,  # 48 - 4f 
+        0,0,0,0,0,0,0,0,  # 50 - 57 
+        0,0,0,0,0,0,0,0,  # 58 - 5f 
+        0,0,0,0,0,0,0,0,  # 60 - 67 
+        0,0,0,0,0,0,0,0,  # 68 - 6f 
+        0,0,0,0,0,0,0,0,  # 70 - 77 
+        0,0,0,0,0,0,0,0,  # 78 - 7f 
+        0,0,0,0,0,0,0,0,  # 80 - 87 
+        0,0,0,0,0,0,0,0,  # 88 - 8f 
+        0,0,0,0,0,0,0,0,  # 90 - 97 
+        0,0,0,0,0,0,0,0,  # 98 - 9f 
+        0,0,0,0,0,0,0,0,  # a0 - a7 
+        0,0,0,0,0,0,0,0,  # a8 - af 
+        0,0,0,0,0,0,0,0,  # b0 - b7 
+        0,0,0,0,0,0,0,0,  # b8 - bf 
+        0,0,0,0,0,0,0,0,  # c0 - c7 
+        0,0,0,0,0,0,0,0,  # c8 - cf 
+        0,0,0,0,0,0,0,0,  # d0 - d7 
+        0,0,0,0,0,0,0,0,  # d8 - df 
+        0,0,0,0,0,0,0,0,  # e0 - e7 
+        0,0,0,0,0,0,0,0,  # e8 - ef 
+        0,0,0,0,0,0,0,0,  # f0 - f7 
+        0,0,0,0,0,0,4,5]  # f8 - ff 
+
+    UCS2BE_st  = [ \
+              5,     7,     7,:Error,     4,     3,:Error,:Error,#00-07 
+         :Error,:Error,:Error,:Error,:ItsMe,:ItsMe,:ItsMe,:ItsMe,#08-0f 
+         :ItsMe,:ItsMe,     6,     6,     6,     6,:Error,:Error,#10-17 
+              6,     6,     6,     6,     6,:ItsMe,     6,     6,#18-1f 
+              6,     6,     6,     6,     5,     7,     7,:Error,#20-27 
+              5,     8,     6,     6,:Error,     6,     6,     6,#28-2f 
+              6,     6,     6,     6,:Error,:Error,:Start,:Start]#30-37 
+
+    UCS2BECharLenTable = [2, 2, 2, 0, 2, 2]
+
+    UCS2BESMModel = {'classTable' =>  UCS2BE_cls,
+                     'classFactor' =>  6,
+                     'stateTable' =>  UCS2BE_st,
+                     'charLenTable' =>  UCS2BECharLenTable,
+                     'name' =>  'UTF-16BE'}
+
+    # UCS2-LE
+
+    UCS2LE_cls = [ \
+        0,0,0,0,0,0,0,0,  # 00 - 07 
+        0,0,1,0,0,2,0,0,  # 08 - 0f 
+        0,0,0,0,0,0,0,0,  # 10 - 17 
+        0,0,0,3,0,0,0,0,  # 18 - 1f 
+        0,0,0,0,0,0,0,0,  # 20 - 27 
+        0,3,3,3,3,3,0,0,  # 28 - 2f 
+        0,0,0,0,0,0,0,0,  # 30 - 37 
+        0,0,0,0,0,0,0,0,  # 38 - 3f 
+        0,0,0,0,0,0,0,0,  # 40 - 47 
+        0,0,0,0,0,0,0,0,  # 48 - 4f 
+        0,0,0,0,0,0,0,0,  # 50 - 57 
+        0,0,0,0,0,0,0,0,  # 58 - 5f 
+        0,0,0,0,0,0,0,0,  # 60 - 67 
+        0,0,0,0,0,0,0,0,  # 68 - 6f 
+        0,0,0,0,0,0,0,0,  # 70 - 77 
+        0,0,0,0,0,0,0,0,  # 78 - 7f 
+        0,0,0,0,0,0,0,0,  # 80 - 87 
+        0,0,0,0,0,0,0,0,  # 88 - 8f 
+        0,0,0,0,0,0,0,0,  # 90 - 97 
+        0,0,0,0,0,0,0,0,  # 98 - 9f 
+        0,0,0,0,0,0,0,0,  # a0 - a7 
+        0,0,0,0,0,0,0,0,  # a8 - af 
+        0,0,0,0,0,0,0,0,  # b0 - b7 
+        0,0,0,0,0,0,0,0,  # b8 - bf 
+        0,0,0,0,0,0,0,0,  # c0 - c7 
+        0,0,0,0,0,0,0,0,  # c8 - cf 
+        0,0,0,0,0,0,0,0,  # d0 - d7 
+        0,0,0,0,0,0,0,0,  # d8 - df 
+        0,0,0,0,0,0,0,0,  # e0 - e7 
+        0,0,0,0,0,0,0,0,  # e8 - ef 
+        0,0,0,0,0,0,0,0,  # f0 - f7 
+        0,0,0,0,0,0,4,5]  # f8 - ff 
+
+    UCS2LE_st = [ \
+              6,     6,     7,     6,     4,     3,:Error,:Error,#00-07 
+         :Error,:Error,:Error,:Error,:ItsMe,:ItsMe,:ItsMe,:ItsMe,#08-0f 
+         :ItsMe,:ItsMe,     5,     5,     5,:Error,:ItsMe,:Error,#10-17 
+              5,     5,     5,:Error,     5,:Error,     6,     6,#18-1f 
+              7,     6,     8,     8,     5,     5,     5,:Error,#20-27 
+              5,     5,     5,:Error,:Error,:Error,     5,     5,#28-2f 
+              5,     5,     5,:Error,     5,:Error,:Start,:Start]#30-37 
+
+    UCS2LECharLenTable = [2, 2, 2, 2, 2, 2]
+
+    UCS2LESMModel = {'classTable' =>  UCS2LE_cls,
+                     'classFactor' =>  6,
+                     'stateTable' =>  UCS2LE_st,
+                     'charLenTable' =>  UCS2LECharLenTable,
+                     'name' =>  'UTF-16LE'}
+
+    # UTF-8
+
+    UTF8_cls = [ \
+        1,1,1,1,1,1,1,1,  # 00 - 07  #allow 0x00 as a legal value
+        1,1,1,1,1,1,0,0,  # 08 - 0f 
+        1,1,1,1,1,1,1,1,  # 10 - 17 
+        1,1,1,0,1,1,1,1,  # 18 - 1f 
+        1,1,1,1,1,1,1,1,  # 20 - 27 
+        1,1,1,1,1,1,1,1,  # 28 - 2f 
+        1,1,1,1,1,1,1,1,  # 30 - 37 
+        1,1,1,1,1,1,1,1,  # 38 - 3f 
+        1,1,1,1,1,1,1,1,  # 40 - 47 
+        1,1,1,1,1,1,1,1,  # 48 - 4f 
+        1,1,1,1,1,1,1,1,  # 50 - 57 
+        1,1,1,1,1,1,1,1,  # 58 - 5f 
+        1,1,1,1,1,1,1,1,  # 60 - 67 
+        1,1,1,1,1,1,1,1,  # 68 - 6f 
+        1,1,1,1,1,1,1,1,  # 70 - 77 
+        1,1,1,1,1,1,1,1,  # 78 - 7f 
+        2,2,2,2,3,3,3,3,  # 80 - 87 
+        4,4,4,4,4,4,4,4,  # 88 - 8f 
+        4,4,4,4,4,4,4,4,  # 90 - 97 
+        4,4,4,4,4,4,4,4,  # 98 - 9f 
+        5,5,5,5,5,5,5,5,  # a0 - a7 
+        5,5,5,5,5,5,5,5,  # a8 - af 
+        5,5,5,5,5,5,5,5,  # b0 - b7 
+        5,5,5,5,5,5,5,5,  # b8 - bf 
+        0,0,6,6,6,6,6,6,  # c0 - c7 
+        6,6,6,6,6,6,6,6,  # c8 - cf 
+        6,6,6,6,6,6,6,6,  # d0 - d7 
+        6,6,6,6,6,6,6,6,  # d8 - df 
+        7,8,8,8,8,8,8,8,  # e0 - e7 
+        8,8,8,8,8,9,8,8,  # e8 - ef 
+        10,11,11,11,11,11,11,11,  # f0 - f7 
+        12,13,13,13,14,15,0,0]   # f8 - ff 
+
+    UTF8_st = [ \
+        :Error,:Start,:Error,:Error,:Error,:Error,     12,   10,#00-07 
+             9,     11,     8,     7,     6,     5,     4,    3,#08-0f 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#10-17 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#18-1f 
+        :ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,#20-27 
+        :ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,:ItsMe,#28-2f 
+        :Error,:Error,     5,     5,     5,     5,:Error,:Error,#30-37 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#38-3f 
+        :Error,:Error,:Error,     5,     5,     5,:Error,:Error,#40-47 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#48-4f 
+        :Error,:Error,     7,     7,     7,     7,:Error,:Error,#50-57 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#58-5f 
+        :Error,:Error,:Error,:Error,     7,     7,:Error,:Error,#60-67 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#68-6f 
+        :Error,:Error,     9,     9,     9,     9,:Error,:Error,#70-77 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#78-7f 
+        :Error,:Error,:Error,:Error,:Error,     9,:Error,:Error,#80-87 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#88-8f 
+        :Error,:Error,    12,    12,    12,    12,:Error,:Error,#90-97 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#98-9f 
+        :Error,:Error,:Error,:Error,:Error,    12,:Error,:Error,#a0-a7 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#a8-af 
+        :Error,:Error,    12,    12,    12,:Error,:Error,:Error,#b0-b7 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error,#b8-bf 
+        :Error,:Error,:Start,:Start,:Start,:Start,:Error,:Error,#c0-c7 
+        :Error,:Error,:Error,:Error,:Error,:Error,:Error,:Error]#c8-cf 
+
+    UTF8CharLenTable = [0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6]
+
+    UTF8SMModel = {'classTable' =>  UTF8_cls,
+                   'classFactor' =>  16,
+                   'stateTable' =>  UTF8_st,
+                   'charLenTable' =>  UTF8CharLenTable,
+                   'name' =>  'UTF-8'}
+end