lg_pod_plugin 1.0.4 → 1.0.6

data/lib/rchardet/latin1prober.rb

@@ -0,0 +1,147 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  FREQ_CAT_NUM = 4
+
+  UDF = 0 # undefined
+  OTH = 1 # other
+  ASC = 2 # ascii capital letter
+  ASS = 3 # ascii small letter
+  ACV = 4 # accent capital vowel
+  ACO = 5 # accent capital other
+  ASV = 6 # accent small vowel
+  ASO = 7 # accent small other
+  CLASS_NUM = 8 # total classes
+
+  Latin1_CharToClass = [
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 00 - 07
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 08 - 0F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 10 - 17
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 18 - 1F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 20 - 27
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 28 - 2F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 30 - 37
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 38 - 3F
+    OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 40 - 47
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 48 - 4F
+    ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   # 50 - 57
+    ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   # 58 - 5F
+    OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 60 - 67
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 68 - 6F
+    ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   # 70 - 77
+    ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   # 78 - 7F
+    OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   # 80 - 87
+    OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   # 88 - 8F
+    UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # 90 - 97
+    OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   # 98 - 9F
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A0 - A7
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # A8 - AF
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B0 - B7
+    OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   # B8 - BF
+    ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   # C0 - C7
+    ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   # C8 - CF
+    ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   # D0 - D7
+    ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   # D8 - DF
+    ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   # E0 - E7
+    ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   # E8 - EF
+    ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   # F0 - F7
+    ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO,   # F8 - FF
+  ].freeze
+
+  # 0 : illegal 
+  # 1 : very unlikely 
+  # 2 : normal 
+  # 3 : very likely
+  Latin1ClassModel = [
+    # UDF OTH ASC ASS ACV ACO ASV ASO
+    0,  0,  0,  0,  0,  0,  0,  0,  # UDF
+    0,  3,  3,  3,  3,  3,  3,  3,  # OTH
+    0,  3,  3,  3,  3,  3,  3,  3,  # ASC
+    0,  3,  3,  3,  1,  1,  3,  3,  # ASS
+    0,  3,  3,  3,  1,  2,  1,  2,  # ACV
+    0,  3,  3,  3,  3,  3,  3,  3,  # ACO
+    0,  3,  1,  3,  1,  1,  1,  3,  # ASV
+    0,  3,  1,  3,  1,  1,  3,  3,  # ASO
+  ].freeze
+
+  class Latin1Prober < CharSetProber
+    def initialize
+      super
+      reset()
+    end
+
+    def reset
+      @lastCharClass = OTH
+      @freqCounter = [0] * FREQ_CAT_NUM
+      super
+    end
+
+    def get_charset_name
+      return "windows-1252"
+    end
+
+    def feed(aBuf)
+      aBuf = filter_with_english_letters(aBuf)
+      aBuf.each_byte do |b|
+        c = b.chr
+        charClass = Latin1_CharToClass[c.bytes.first]
+        freq = Latin1ClassModel[(@lastCharClass * CLASS_NUM) + charClass]
+        if freq == 0
+          @state = ENotMe
+          break
+        end
+        @freqCounter[freq] += 1
+        @lastCharClass = charClass
+      end
+
+      return get_state()
+    end
+
+    def get_confidence
+      if get_state() == ENotMe
+        return 0.01
+      end
+
+      total = @freqCounter.inject{|a,b| a+b} 
+      if total < 0.01
+        confidence = 0.0
+      else
+        confidence = (@freqCounter[3] / total) - (@freqCounter[1] * 20.0 / total)
+      end
+      if confidence < 0.0
+        confidence = 0.0
+      end
+      # lower the confidence of latin1 so that other more accurate detector 
+      # can take priority.
+      confidence = confidence * 0.5
+      return confidence
+    end
+  end
+end

data/lib/rchardet/mbcharsetprober.rb

@@ -0,0 +1,89 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#   Proofpoint, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class MultiByteCharSetProber < CharSetProber
+    def initialize
+      super
+      @distributionAnalyzer = nil
+      @codingSM = nil
+      @lastChar = "\x00\x00"
+    end
+
+    def reset
+      super
+      if @codingSM
+        @codingSM.reset()
+      end
+      if @distributionAnalyzer
+        @distributionAnalyzer.reset()
+      end
+      @lastChar = "\x00\x00"
+    end
+
+    def get_charset_name
+    end
+
+    def feed(aBuf)
+      aLen = aBuf.length
+      for i in (0...aLen)
+        codingState = @codingSM.next_state(aBuf[i, 1])
+        if codingState == EError
+          $stderr << "#{get_charset_name} prober hit error at byte #{i}\n" if $debug
+          @state = ENotMe
+          break
+        elsif codingState == EItsMe
+          @state = EFoundIt
+          break
+        elsif codingState == EStart
+          charLen = @codingSM.get_current_charlen()
+          if i == 0
+            @lastChar[1] = aBuf[0, 1]
+            @distributionAnalyzer.feed(@lastChar, charLen)
+          else
+            @distributionAnalyzer.feed(aBuf[i-1, 2], charLen)
+          end
+        end
+      end
+      @lastChar[0] = aBuf[aLen-1, 1]
+
+      if get_state() == EDetecting
+        if @distributionAnalyzer.got_enough_data() and (get_confidence() > SHORTCUT_THRESHOLD)
+          @state = EFoundIt
+        end
+      end
+      return get_state()
+    end
+
+    def get_confidence
+      return @distributionAnalyzer.get_confidence()
+    end
+  end
+end

data/lib/rchardet/mbcsgroupprober.rb

@@ -0,0 +1,47 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#   Proofpoint, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class MBCSGroupProber < CharSetGroupProber
+    def initialize
+      super
+      @probers = [ 
+                    UTF8Prober.new,
+                    SJISProber.new,
+                    EUCJPProber.new,
+                    GB18030Prober.new,
+                    EUCKRProber.new,
+                    Big5Prober.new,
+                    EUCTWProber.new
+                   ]
+      reset()
+    end
+  end
+end