rchardet 0.9.9

data/lib/chardet/codingstatemachine.rb

@@ -0,0 +1,64 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CodingStateMachine
+    def initialize(sm)
+      @_mModel = sm
+      @_mCurrentBytePos = 0
+      @_mCurrentCharLen = 0
+      reset()
+    end
+
+    def reset
+      @_mCurrentState = EStart
+    end
+
+    def next_state(c)
+      # for each byte we get its class
+      # if it is first byte, we also get byte length
+      byteCls = @_mModel['classTable'][c[0]]
+      if @_mCurrentState == EStart
+	@_mCurrentBytePos = 0
+	@_mCurrentCharLen = @_mModel['charLenTable'][byteCls]
+      end
+      # from byte's class and stateTable, we get its next state
+      @_mCurrentState = @_mModel['stateTable'][@_mCurrentState * @_mModel['classFactor'] + byteCls]
+      @_mCurrentBytePos += 1
+      return @_mCurrentState
+    end
+
+    def get_current_charlen
+      return @_mCurrentCharLen
+    end
+
+    def get_coding_state_machine
+      return @_mModel['name']
+    end
+  end
+end

data/lib/chardet/constants.rb

@@ -0,0 +1,42 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  $debug = false
+
+  EDetecting = 0
+  EFoundIt = 1
+  ENotMe = 2
+
+  EStart = 0
+  EError = 1
+  EItsMe = 2
+
+  SHORTCUT_THRESHOLD = 0.95
+end

data/lib/chardet/escprober.rb

@@ -0,0 +1,89 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class EscCharSetProber < CharSetProber
+    def initialize
+      super()
+      @_mCodingSM = [ 
+	CodingStateMachine.new(HZSMModel),
+	CodingStateMachine.new(ISO2022CNSMModel),
+	CodingStateMachine.new(ISO2022JPSMModel),
+	CodingStateMachine.new(ISO2022KRSMModel)
+      ]
+      reset()
+    end
+
+    def reset
+      super()
+      for codingSM in @_mCodingSM:
+	next if not codingSM
+	codingSM.active = true
+	codingSM.reset()
+      end
+      @_mActiveSM = @_mCodingSM.length
+      @_mDetectedCharset = nil
+    end
+
+    def get_charset_name
+      return @_mDetectedCharset
+    end
+
+    def get_confidence
+      if @_mDetectedCharset
+	return 0.99
+      else
+	return 0.00
+      end
+    end
+
+    def feed(aBuf)
+      for c in aBuf.split('')
+	for codingSM in @_mCodingSM
+	  next unless codingSM
+	  next unless codingSM.active
+	  codingState = codingSM.next_state(c)
+	  if codingState == EError
+	    codingSM.active = false
+	    @_mActiveSM -= 1
+	    if @_mActiveSM <= 0
+	      @_mState = ENotMe
+	      return get_state()
+	    end
+	  elsif codingState == EItsMe
+	    @_mState = EFoundIt
+	    @_mDetectedCharset = codingSM.get_coding_state_machine()
+	    return get_state()
+	  end
+	end
+      end
+
+      return get_state()
+    end
+  end
+end

data/lib/chardet/escsm.rb

@@ -0,0 +1,244 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  HZ_cls = [
+    1,0,0,0,0,0,0,0,  # 00 - 07 
+    0,0,0,0,0,0,0,0,  # 08 - 0f 
+    0,0,0,0,0,0,0,0,  # 10 - 17 
+    0,0,0,1,0,0,0,0,  # 18 - 1f 
+    0,0,0,0,0,0,0,0,  # 20 - 27 
+    0,0,0,0,0,0,0,0,  # 28 - 2f 
+    0,0,0,0,0,0,0,0,  # 30 - 37 
+    0,0,0,0,0,0,0,0,  # 38 - 3f 
+    0,0,0,0,0,0,0,0,  # 40 - 47 
+    0,0,0,0,0,0,0,0,  # 48 - 4f 
+    0,0,0,0,0,0,0,0,  # 50 - 57 
+    0,0,0,0,0,0,0,0,  # 58 - 5f 
+    0,0,0,0,0,0,0,0,  # 60 - 67 
+    0,0,0,0,0,0,0,0,  # 68 - 6f 
+    0,0,0,0,0,0,0,0,  # 70 - 77 
+    0,0,0,4,0,5,2,0,  # 78 - 7f 
+    1,1,1,1,1,1,1,1,  # 80 - 87 
+    1,1,1,1,1,1,1,1,  # 88 - 8f 
+    1,1,1,1,1,1,1,1,  # 90 - 97 
+    1,1,1,1,1,1,1,1,  # 98 - 9f 
+    1,1,1,1,1,1,1,1,  # a0 - a7 
+    1,1,1,1,1,1,1,1,  # a8 - af 
+    1,1,1,1,1,1,1,1,  # b0 - b7 
+    1,1,1,1,1,1,1,1,  # b8 - bf 
+    1,1,1,1,1,1,1,1,  # c0 - c7 
+    1,1,1,1,1,1,1,1,  # c8 - cf 
+    1,1,1,1,1,1,1,1,  # d0 - d7 
+    1,1,1,1,1,1,1,1,  # d8 - df 
+    1,1,1,1,1,1,1,1,  # e0 - e7 
+    1,1,1,1,1,1,1,1,  # e8 - ef 
+    1,1,1,1,1,1,1,1,  # f0 - f7 
+    1,1,1,1,1,1,1,1,  # f8 - ff 
+  ]
+
+  HZ_st = [
+    EStart,EError,     3,EStart,EStart,EStart,EError,EError,# 00-07 
+    EError,EError,EError,EError,EItsMe,EItsMe,EItsMe,EItsMe,# 08-0f 
+    EItsMe,EItsMe,EError,EError,EStart,EStart,     4,EError,# 10-17 
+    5,EError,     6,EError,     5,     5,     4,EError,# 18-1f 
+    4,EError,     4,     4,     4,EError,     4,EError,# 20-27 
+    4,EItsMe,EStart,EStart,EStart,EStart,EStart,EStart,# 28-2f 
+  ]
+
+  HZCharLenTable = [0, 0, 0, 0, 0, 0]
+
+  HZSMModel = {'classTable' => HZ_cls,
+	     'classFactor' => 6,
+	     'stateTable' => HZ_st,
+	     'charLenTable' => HZCharLenTable,
+	     'name' => "HZ-GB-2312"
+}
+
+ISO2022CN_cls = [
+2,0,0,0,0,0,0,0,  # 00 - 07 
+0,0,0,0,0,0,0,0,  # 08 - 0f 
+0,0,0,0,0,0,0,0,  # 10 - 17 
+0,0,0,1,0,0,0,0,  # 18 - 1f 
+0,0,0,0,0,0,0,0,  # 20 - 27 
+0,3,0,0,0,0,0,0,  # 28 - 2f 
+0,0,0,0,0,0,0,0,  # 30 - 37 
+0,0,0,0,0,0,0,0,  # 38 - 3f 
+0,0,0,4,0,0,0,0,  # 40 - 47 
+0,0,0,0,0,0,0,0,  # 48 - 4f 
+0,0,0,0,0,0,0,0,  # 50 - 57 
+0,0,0,0,0,0,0,0,  # 58 - 5f 
+0,0,0,0,0,0,0,0,  # 60 - 67 
+0,0,0,0,0,0,0,0,  # 68 - 6f 
+0,0,0,0,0,0,0,0,  # 70 - 77 
+0,0,0,0,0,0,0,0,  # 78 - 7f 
+2,2,2,2,2,2,2,2,  # 80 - 87 
+2,2,2,2,2,2,2,2,  # 88 - 8f 
+2,2,2,2,2,2,2,2,  # 90 - 97 
+2,2,2,2,2,2,2,2,  # 98 - 9f 
+2,2,2,2,2,2,2,2,  # a0 - a7 
+2,2,2,2,2,2,2,2,  # a8 - af 
+2,2,2,2,2,2,2,2,  # b0 - b7 
+2,2,2,2,2,2,2,2,  # b8 - bf 
+2,2,2,2,2,2,2,2,  # c0 - c7 
+2,2,2,2,2,2,2,2,  # c8 - cf 
+2,2,2,2,2,2,2,2,  # d0 - d7 
+2,2,2,2,2,2,2,2,  # d8 - df 
+2,2,2,2,2,2,2,2,  # e0 - e7 
+2,2,2,2,2,2,2,2,  # e8 - ef 
+2,2,2,2,2,2,2,2,  # f0 - f7 
+2,2,2,2,2,2,2,2,  # f8 - ff 
+]
+
+ISO2022CN_st = [
+EStart,     3,EError,EStart,EStart,EStart,EStart,EStart,# 00-07 
+EStart,EError,EError,EError,EError,EError,EError,EError,# 08-0f 
+EError,EError,EItsMe,EItsMe,EItsMe,EItsMe,EItsMe,EItsMe,# 10-17 
+EItsMe,EItsMe,EItsMe,EError,EError,EError,     4,EError,# 18-1f 
+EError,EError,EError,EItsMe,EError,EError,EError,EError,# 20-27 
+     5,     6,EError,EError,EError,EError,EError,EError,# 28-2f 
+EError,EError,EError,EItsMe,EError,EError,EError,EError,# 30-37 
+EError,EError,EError,EError,EError,EItsMe,EError,EStart,# 38-3f 
+]
+
+ISO2022CNCharLenTable = [0, 0, 0, 0, 0, 0, 0, 0, 0]
+
+ISO2022CNSMModel = {'classTable' => ISO2022CN_cls,
+		    'classFactor' => 9,
+		    'stateTable' => ISO2022CN_st,
+		    'charLenTable' => ISO2022CNCharLenTable,
+		    'name' => "ISO-2022-CN"
+}
+
+ISO2022JP_cls = [
+2,0,0,0,0,0,0,0,  # 00 - 07 
+0,0,0,0,0,0,2,2,  # 08 - 0f 
+0,0,0,0,0,0,0,0,  # 10 - 17 
+0,0,0,1,0,0,0,0,  # 18 - 1f 
+0,0,0,0,7,0,0,0,  # 20 - 27 
+3,0,0,0,0,0,0,0,  # 28 - 2f 
+0,0,0,0,0,0,0,0,  # 30 - 37 
+0,0,0,0,0,0,0,0,  # 38 - 3f 
+6,0,4,0,8,0,0,0,  # 40 - 47 
+0,9,5,0,0,0,0,0,  # 48 - 4f 
+0,0,0,0,0,0,0,0,  # 50 - 57 
+0,0,0,0,0,0,0,0,  # 58 - 5f 
+0,0,0,0,0,0,0,0,  # 60 - 67 
+0,0,0,0,0,0,0,0,  # 68 - 6f 
+0,0,0,0,0,0,0,0,  # 70 - 77 
+0,0,0,0,0,0,0,0,  # 78 - 7f 
+2,2,2,2,2,2,2,2,  # 80 - 87 
+2,2,2,2,2,2,2,2,  # 88 - 8f 
+2,2,2,2,2,2,2,2,  # 90 - 97 
+2,2,2,2,2,2,2,2,  # 98 - 9f 
+2,2,2,2,2,2,2,2,  # a0 - a7 
+2,2,2,2,2,2,2,2,  # a8 - af 
+2,2,2,2,2,2,2,2,  # b0 - b7 
+2,2,2,2,2,2,2,2,  # b8 - bf 
+2,2,2,2,2,2,2,2,  # c0 - c7 
+2,2,2,2,2,2,2,2,  # c8 - cf 
+2,2,2,2,2,2,2,2,  # d0 - d7 
+2,2,2,2,2,2,2,2,  # d8 - df 
+2,2,2,2,2,2,2,2,  # e0 - e7 
+2,2,2,2,2,2,2,2,  # e8 - ef 
+2,2,2,2,2,2,2,2,  # f0 - f7 
+2,2,2,2,2,2,2,2,  # f8 - ff 
+]
+
+ISO2022JP_st = [ 
+EStart,     3,EError,EStart,EStart,EStart,EStart,EStart,# 00-07 
+EStart,EStart,EError,EError,EError,EError,EError,EError,# 08-0f 
+EError,EError,EError,EError,EItsMe,EItsMe,EItsMe,EItsMe,# 10-17 
+EItsMe,EItsMe,EItsMe,EItsMe,EItsMe,EItsMe,EError,EError,# 18-1f 
+EError,     5,EError,EError,EError,     4,EError,EError,# 20-27 
+EError,EError,EError,     6,EItsMe,EError,EItsMe,EError,# 28-2f 
+EError,EError,EError,EError,EError,EError,EItsMe,EItsMe,# 30-37 
+EError,EError,EError,EItsMe,EError,EError,EError,EError,# 38-3f 
+EError,EError,EError,EError,EItsMe,EError,EStart,EStart,# 40-47 
+]
+
+ISO2022JPCharLenTable = [0, 0, 0, 0, 0, 0, 0, 0]
+
+ISO2022JPSMModel = {'classTable' => ISO2022JP_cls,
+		    'classFactor' => 10,
+		    'stateTable' => ISO2022JP_st,
+		    'charLenTable' => ISO2022JPCharLenTable,
+		    'name' => "ISO-2022-JP"
+}
+
+ISO2022KR_cls = [
+2,0,0,0,0,0,0,0,  # 00 - 07 
+0,0,0,0,0,0,0,0,  # 08 - 0f 
+0,0,0,0,0,0,0,0,  # 10 - 17 
+0,0,0,1,0,0,0,0,  # 18 - 1f 
+0,0,0,0,3,0,0,0,  # 20 - 27 
+0,4,0,0,0,0,0,0,  # 28 - 2f 
+0,0,0,0,0,0,0,0,  # 30 - 37 
+0,0,0,0,0,0,0,0,  # 38 - 3f 
+0,0,0,5,0,0,0,0,  # 40 - 47 
+0,0,0,0,0,0,0,0,  # 48 - 4f 
+0,0,0,0,0,0,0,0,  # 50 - 57 
+0,0,0,0,0,0,0,0,  # 58 - 5f 
+0,0,0,0,0,0,0,0,  # 60 - 67 
+0,0,0,0,0,0,0,0,  # 68 - 6f 
+0,0,0,0,0,0,0,0,  # 70 - 77 
+0,0,0,0,0,0,0,0,  # 78 - 7f 
+2,2,2,2,2,2,2,2,  # 80 - 87 
+2,2,2,2,2,2,2,2,  # 88 - 8f 
+2,2,2,2,2,2,2,2,  # 90 - 97 
+2,2,2,2,2,2,2,2,  # 98 - 9f 
+2,2,2,2,2,2,2,2,  # a0 - a7 
+2,2,2,2,2,2,2,2,  # a8 - af 
+2,2,2,2,2,2,2,2,  # b0 - b7 
+2,2,2,2,2,2,2,2,  # b8 - bf 
+2,2,2,2,2,2,2,2,  # c0 - c7 
+2,2,2,2,2,2,2,2,  # c8 - cf 
+2,2,2,2,2,2,2,2,  # d0 - d7 
+2,2,2,2,2,2,2,2,  # d8 - df 
+2,2,2,2,2,2,2,2,  # e0 - e7 
+2,2,2,2,2,2,2,2,  # e8 - ef 
+2,2,2,2,2,2,2,2,  # f0 - f7 
+2,2,2,2,2,2,2,2,  # f8 - ff 
+]
+
+ISO2022KR_st = [ 
+EStart,     3,EError,EStart,EStart,EStart,EError,EError,# 00-07 
+EError,EError,EError,EError,EItsMe,EItsMe,EItsMe,EItsMe,# 08-0f 
+EItsMe,EItsMe,EError,EError,EError,     4,EError,EError,# 10-17 
+EError,EError,EError,EError,     5,EError,EError,EError,# 18-1f 
+EError,EError,EError,EItsMe,EStart,EStart,EStart,EStart,# 20-27 
+]
+
+ISO2022KRCharLenTable = [0, 0, 0, 0, 0, 0]
+
+ISO2022KRSMModel = {'classTable' => ISO2022KR_cls,
+		    'classFactor' => 6,
+		    'stateTable' => ISO2022KR_st,
+		    'charLenTable' => ISO2022KRCharLenTable,
+		    'name' => "ISO-2022-KR"
+}
+end

data/lib/chardet/eucjpprober.rb

@@ -0,0 +1,88 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class EUCJPProber < MultiByteCharSetProber
+    def initialize
+      super()
+      @_mCodingSM = CodingStateMachine.new(EUCJPSMModel)
+      @_mDistributionAnalyzer = EUCJPDistributionAnalysis.new()
+      @_mContextAnalyzer = EUCJPContextAnalysis.new()
+      reset
+    end
+
+    def reset
+      super()
+      @_mContextAnalyzer.reset()
+    end
+
+    def get_charset_name
+      return "EUC-JP"
+    end
+
+    def feed(aBuf)
+      aLen = aBuf.length
+      for i in (0...aLen)
+	codingState = @_mCodingSM.next_state(aBuf[i..i])
+	if codingState == EError
+	  $stderr << "#{get_charset_name} prober hit error at byte #{i}\n" if $debug
+	  @_mState = ENotMe
+	  break
+	elsif codingState == EItsMe
+	  @_mState = EFoundIt
+	  break
+	elsif codingState == EStart:
+	  charLen = @_mCodingSM.get_current_charlen()
+	  if i == 0
+	    @_mLastChar[1] = aBuf[0..0]
+	    @_mContextAnalyzer.feed(@_mLastChar, charLen)
+	    @_mDistributionAnalyzer.feed(@_mLastChar, charLen)
+	  else
+	    @_mContextAnalyzer.feed(aBuf[i-1...i+1], charLen)
+	    @_mDistributionAnalyzer.feed(aBuf[i-1...i+1], charLen)
+	  end
+	end
+      end
+
+      @_mLastChar[0] = aBuf[aLen-1..aLen-1]
+
+      if get_state() == EDetecting
+	if @_mContextAnalyzer.got_enough_data() and (get_confidence() > SHORTCUT_THRESHOLD)
+	  @_mState = EFoundIt
+	end
+      end
+
+      return get_state()
+    end
+
+    def get_confidence
+      l = [@_mContextAnalyzer.get_confidence,@_mDistributionAnalyzer.get_confidence]
+      return l.max
+    end
+  end
+end