chardet2 1.0.0

data/lib/Latin1Prober.rb

@@ -0,0 +1,155 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetProber'
+
+module Enumerable
+    def reduceBlock(res)
+        each { |n| res = yield(res, n) }
+        res
+    end
+end
+
+module UniversalDetector
+    FREQ_CAT_NUM = 4
+
+    UDF = 0 # undefined
+    OTH = 1 # other
+    ASC = 2 # ascii capital letter
+    ASS = 3 # ascii small letter
+    ACV = 4 # accent capital vowel
+    ACO = 5 # accent capital other
+    ASV = 6 # accent small vowel
+    ASO = 7 # accent small other
+    CLASS_NUM = 8 # total classes
+
+    Latin1_CharToClass = [ \
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 00 - 07
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 08 - 0F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 10 - 17
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 18 - 1F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 20 - 27
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 28 - 2F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 30 - 37
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 38 - 3F
+      OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 40 - 47
+      ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 48 - 4F
+      ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 50 - 57
+      ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   # 58 - 5F
+      OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 60 - 67
+      ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 68 - 6F
+      ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 70 - 77
+      ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   # 78 - 7F
+      OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   # 80 - 87
+      OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   # 88 - 8F
+      UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 90 - 97
+      OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   # 98 - 9F
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A0 - A7
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A8 - AF
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B0 - B7
+      OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B8 - BF
+      ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   # C0 - C7
+      ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   # C8 - CF
+      ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   # D0 - D7
+      ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   # D8 - DF
+      ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   # E0 - E7
+      ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   # E8 - EF
+      ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   # F0 - F7
+      ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO,   # F8 - FF
+    ]
+
+    # 0 : illegal 
+    # 1 : very unlikely 
+    # 2 : normal 
+    # 3 : very likely
+    Latin1ClassModel = [ \
+    # UDF OTH ASC ASS ACV ACO ASV ASO
+       0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+       0,  3,  3,  3,  3,  3,  3,  3,  # OTH
+       0,  3,  3,  3,  3,  3,  3,  3,  # ASC
+       0,  3,  3,  3,  1,  1,  3,  3,  # ASS
+       0,  3,  3,  3,  1,  2,  1,  2,  # ACV
+       0,  3,  3,  3,  3,  3,  3,  3,  # ACO
+       0,  3,  1,  3,  1,  1,  1,  3,  # ASV
+       0,  3,  1,  3,  1,  1,  3,  3,  # ASO
+    ]
+
+    class Latin1Prober < CharSetProber
+        def initialize
+            super
+            reset()
+        end
+
+        def reset
+            @_mLastCharClass = OTH
+            @_mFreqCounter = [0] * FREQ_CAT_NUM
+            super
+        end
+
+        def get_charset_name
+            return "windows-1252"
+        end
+
+        def feed(aBuf)
+            aBuf = filter_with_english_letters(aBuf)
+            for c in aBuf
+                charClass = Latin1_CharToClass[c[0]]
+                freq = Latin1ClassModel[(@_mLastCharClass * CLASS_NUM) + charClass]
+                if freq == 0
+                    @_mState = :NotMe
+                    break
+                end
+                @_mFreqCounter[freq] += 1
+                @_mLastCharClass = charClass
+            end        
+
+            return get_state()
+        end
+
+        def get_confidence()
+            if get_state() == :NotMe
+                return 0.01
+            end
+
+            total = @_mFreqCounter.reduce(0, :+)
+            if total < 0.01
+                confidence = 0.0
+            else
+                confidence = (@_mFreqCounter[3] / total) - (@_mFreqCounter[1] * 20.0 / total)
+            end
+            if confidence < 0.0
+                confidence = 0.0
+            end
+            # lower the confidence of latin1 so that other more accurate detector 
+            # can take priority.
+            confidence = confidence * 0.5
+            return confidence
+        end
+    end
+end

data/lib/MBCSGroupProber.rb

@@ -0,0 +1,57 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetGroupProber'
+require 'UTF8Prober'
+require 'SJISProber'
+require 'EUCJPProber'
+require 'GB2312Prober'
+require 'EUCKRProber'
+require 'Big5Prober'
+require 'EUCTWProber'
+
+module UniversalDetector
+    class MBCSGroupProber < CharSetGroupProber
+        
+        attr_reader :mProbers
+        
+        def initialize
+            super
+            @mProbers = [ \
+                UTF8Prober.new,
+                SJISProber.new,
+                EUCJPProber.new,
+                GB2312Prober.new,
+                EUCKRProber.new,
+                Big5Prober.new,
+                EUCTWProber.new]
+            reset()
+        end
+    end
+end