chardet 0.9.0

data/lib/MultiByteCharSetProber.rb

@@ -0,0 +1,94 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetProber'
+
+module  UniversalDetector
+
+    class MultiByteCharSetProber < CharSetProber
+        def initialize
+            super
+            @_mDistributionAnalyzer = nil
+            @_mCodingSM = nil
+            @_mLastChar = ['\x00', '\x00']
+        end
+
+        def reset
+            super
+            if @_mCodingSM
+                @_mCodingSM.reset()
+            end
+            if @_mDistributionAnalyzer
+                @_mDistributionAnalyzer.reset()
+            end
+            @_mLastChar = ['\x00', '\x00']
+        end
+
+        def get_charset_name
+        end
+
+        def feed(aBuf)            
+            aLen = aBuf.length
+            for i in 0...aLen
+                codingState = @_mCodingSM.next_state(aBuf[i])
+                if codingState == :Error
+                    if UniversalDetector::DEBUG
+                        p(get_charset_name() + ' prober hit error at byte ' + i.to_s + '\n')
+                    end
+                    @_mState = :NotMe
+                    break                
+                elsif codingState == :ItsMe
+                    @_mState = :FoundIt
+                    break
+                elsif codingState == :Start
+                    charLen = @_mCodingSM.get_current_charlen()
+                    if i == 0
+                        @_mLastChar[1] = aBuf[0]
+                        @_mDistributionAnalyzer.feed(@_mLastChar, charLen)
+                    else
+                        @_mDistributionAnalyzer.feed(aBuf[(i-1)..(i+1)], charLen)
+                    end
+                end
+            end
+
+            @_mLastChar[0] = aBuf[aLen - 1]
+            if get_state() == :Detecting
+                if @_mDistributionAnalyzer.got_enough_data() && (get_confidence() > SHORTCUT_THRESHOLD)
+                    @_mState = :FoundIt
+                end
+            end
+
+            return get_state()
+        end
+
+        def get_confidence
+            return @_mDistributionAnalyzer.get_confidence()
+        end
+    end
+end

data/lib/SBCSGroupProber.rb

@@ -0,0 +1,71 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetGroupProber'
+require 'SingleByteCharSetProber'
+require 'LangCyrillicModel'
+require 'LangGreekModel'
+require 'LangHebrewModel'
+require 'LangHungarianModel'
+require 'LangBulgarianModel'
+require 'LangThaiModel'
+require 'HebrewProber'
+
+module UniversalDetector
+    class SBCSGroupProber < CharSetGroupProber
+        
+        attr_reader :mProbers
+        
+        def initialize
+            super
+            @mProbers = [ \
+                SingleByteCharSetProber.new(Win1251CyrillicModel),
+                SingleByteCharSetProber.new(Koi8rModel),
+                SingleByteCharSetProber.new(Latin5CyrillicModel),
+                SingleByteCharSetProber.new(MacCyrillicModel),
+                SingleByteCharSetProber.new(Ibm866Model),
+                SingleByteCharSetProber.new(Ibm855Model),
+                SingleByteCharSetProber.new(Latin7GreekModel),
+                SingleByteCharSetProber.new(Win1253GreekModel),
+                SingleByteCharSetProber.new(Latin5BulgarianModel),
+                SingleByteCharSetProber.new(Win1251BulgarianModel),
+                SingleByteCharSetProber.new(Latin2HungarianModel),
+                SingleByteCharSetProber.new(Win1250HungarianModel),
+                SingleByteCharSetProber.new(TIS620ThaiModel),
+                ]
+            hebrewProber = HebrewProber.new
+            logicalHebrewProber = SingleByteCharSetProber.new(Win1255HebrewModel, false, hebrewProber)
+            visualHebrewProber = SingleByteCharSetProber.new(Win1255HebrewModel, true, hebrewProber)
+            hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber)
+            @mProbers = @mProbers + [hebrewProber, logicalHebrewProber, visualHebrewProber]
+
+            reset()
+        end
+    end
+end

data/lib/SJISProber.rb

@@ -0,0 +1,99 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'MultiByteCharSetProber'
+require 'CodingStateMachine'
+require 'JapaneseContextAnalysis'
+require 'CharDistributionAnalysis'
+require 'MBCSSM'
+
+module UniversalDetector
+    class SJISProber < MultiByteCharSetProber
+        def initialize
+            super
+            @_mCodingSM = CodingStateMachine.new(SJISSMModel)
+            @_mDistributionAnalyzer = SJISDistributionAnalysis.new
+            @_mContextAnalyzer = SJISContextAnalysis.new
+            reset()
+        end
+
+        def reset
+            super
+            @_mContextAnalyzer.reset()
+        end
+
+        def get_charset_name
+            return "SHIFT_JIS"
+        end
+
+        def feed(aBuf)
+            aLen = aBuf.length
+            for i in 0...aLen
+                codingState = @_mCodingSM.next_state(aBuf[i])
+                if codingState == :Error
+                    if DEBUG
+                        p(get_charset_name() + ' prober hit error at byte ' + i.to_s + '\n')
+                    end
+                    @_mState = :NotMe                    
+                    break                    
+                elsif codingState == :ItsMe
+                    @_mState = :FoundIt
+                    break
+                elsif codingState == :Start
+                    charLen = @_mCodingSM.get_current_charlen()
+                    if i == 0
+                        @_mLastChar[1] = aBuf[0]
+                        @_mContextAnalyzer.feed(@_mLastChar[2 - charLen..@_mLastChar.length], charLen)
+                        @_mDistributionAnalyzer.feed(@_mLastChar, charLen)                        
+                    else
+                        @_mContextAnalyzer.feed(aBuf[i + 1 - charLen .. i + 3 - charLen], charLen)
+                        @_mDistributionAnalyzer.feed(aBuf[i - 1 .. i + 1], charLen)
+                    end
+                end
+            end
+
+            @_mLastChar[0] = aBuf[aLen - 1]
+
+            if get_state() == :Detecting
+                if @_mContextAnalyzer.got_enough_data() and \
+                       (get_confidence() > SHORTCUT_THRESHOLD)
+                    @_mState = :FoundIt
+                end
+            end
+
+            return get_state()
+        end
+
+        def get_confidence
+            contxtCf = @_mContextAnalyzer.get_confidence()
+            distribCf = @_mDistributionAnalyzer.get_confidence()
+            return [contxtCf, distribCf].max
+        end
+    end
+end

data/lib/SingleByteCharSetProber.rb

@@ -0,0 +1,131 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetProber'
+
+module UniversalDetector
+    
+    SAMPLE_SIZE = 64
+    SB_ENOUGH_REL_THRESHOLD = 1024
+    POSITIVE_SHORTCUT_THRESHOLD = 0.95
+    NEGATIVE_SHORTCUT_THRESHOLD = 0.05
+    SYMBOL_CAT_ORDER = 250
+    NUMBER_OF_SEQ_CAT = 4
+    POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
+    
+    class SingleByteCharSetProber < CharSetProber
+        def initialize(model, reversed=false, nameProber=nil)
+            super()
+            @_mModel = model
+            @_mReversed = reversed # TRUE if we need to reverse every pair in the model lookup
+            @_mNameProber = nameProber # Optional auxiliary prober for name decision
+            reset()
+        end
+
+        def reset
+            super
+            @_mLastOrder = 255 # char order of last character
+            @_mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
+            @_mTotalSeqs = 0
+            @_mTotalChar = 0
+            @_mFreqChar = 0 # characters that fall in our sampling range
+        end
+
+        def get_charset_name
+            if @_mNameProber
+                return @_mNameProber.get_charset_name()
+            else
+                return @_mModel['charsetName']
+            end
+        end
+
+        def feed(aBuf)
+            unless @_mModel['keepEnglishLetter']
+                aBuf = filter_without_english_letters(aBuf)
+            end
+            aLen = aBuf.length
+            unless aLen
+                return get_state()
+            end                        
+            
+            for i in 0...aLen
+                c = aBuf[i]
+                order = @_mModel['charToOrderMap'][c]
+                if order < SYMBOL_CAT_ORDER
+                    @_mTotalChar += 1
+                end
+                if order < SAMPLE_SIZE                    
+                    @_mFreqChar += 1
+                    if @_mLastOrder < SAMPLE_SIZE
+                        @_mTotalSeqs += 1
+                        unless @_mReversed
+                            @_mSeqCounters[@_mModel['precedenceMatrix'][(@_mLastOrder * SAMPLE_SIZE) + order]] += 1                        
+                        else # reverse the order of the letters in the lookup
+                            @_mSeqCounters[@_mModel['precedenceMatrix'][(order * SAMPLE_SIZE) + @_mLastOrder]] += 1
+                        end
+                    end
+                end
+                @_mLastOrder = order
+            end
+
+            if get_state() == :Detecting
+                if @_mTotalSeqs > SB_ENOUGH_REL_THRESHOLD
+                    cf = get_confidence()
+                    if cf > POSITIVE_SHORTCUT_THRESHOLD
+                        if DEBUG
+                            p('%s confidence = %s, we have a winner\n' % [@_mModel['charsetName'], cf])
+                        end
+                        @_mState = :FoundIt
+                    elsif cf < NEGATIVE_SHORTCUT_THRESHOLD
+                        if DEBUG
+                            p('%s confidence = %s, below negative shortcut threshhold %s\n' % [@_mModel['charsetName'], cf, NEGATIVE_SHORTCUT_THRESHOLD])
+                        end
+                        @_mState = :NotMe
+                    end
+                end
+            end
+
+            return get_state()
+        end
+
+        def get_confidence
+            r = 0.01
+            if @_mTotalSeqs > 0
+    #            print @_mSeqCounters[POSITIVE_CAT], @_mTotalSeqs, @_mModel['mTypicalPositiveRatio']
+                r = (1.0 * @_mSeqCounters[POSITIVE_CAT]) / @_mTotalSeqs / @_mModel['mTypicalPositiveRatio']
+    #            print r, @_mFreqChar, @_mTotalChar
+                r = r * @_mFreqChar / @_mTotalChar
+                if r >= 1.0
+                    r = 0.99
+                end
+            end
+            return r
+        end
+    end
+end

data/lib/UTF8Prober.rb

@@ -0,0 +1,91 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require 'UniversalDetector'
+require 'CharSetProber'
+require 'CodingStateMachine'
+require 'MBCSSM'
+
+module UniversalDetector
+    ONE_CHAR_PROB = 0.5
+
+    class UTF8Prober < CharSetProber
+        def initialize
+            super()
+            @_mCodingSM = CodingStateMachine.new(UTF8SMModel)
+            reset()
+        end
+
+        def reset
+            super
+            @_mCodingSM.reset()
+            @_mNumOfMBChar = 0
+        end
+
+        def get_charset_name
+            return "utf-8"
+        end
+
+        def feed(aBuf) 
+            aLen = aBuf.length       
+            for i in 0...aLen
+                codingState = @_mCodingSM.next_state(aBuf[i])                
+                if codingState == :Error
+                    @_mState = :NotMe
+                    break
+                elsif codingState == :ItsMe
+                    @_mState = :FoundIt
+                    break
+                elsif codingState == :Start
+                    if @_mCodingSM.get_current_charlen() >= 2:
+                        @_mNumOfMBChar += 1
+                    end
+                end
+            end
+
+            if get_state() == :Detecting
+                if get_confidence() > SHORTCUT_THRESHOLD
+                    @_mState = :FoundIt
+                end
+            end
+            return get_state()
+        end
+
+        def get_confidence
+            unlike = 0.99
+            if @_mNumOfMBChar < 6
+                for i in 0...@_mNumOfMBChar
+                    unlike = unlike * ONE_CHAR_PROB
+                end
+                return 1.0 - unlike
+            else
+                return unlike
+            end
+        end
+    end
+end