lg_pod_plugin 1.0.4 → 1.0.6

data/lib/rchardet/big5prober.rb

@@ -0,0 +1,42 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class Big5Prober < MultiByteCharSetProber 
+    def initialize
+      super
+      @codingSM = CodingStateMachine.new(Big5SMModel)
+      @distributionAnalyzer = Big5DistributionAnalysis.new()
+      reset()
+    end
+
+    def get_charset_name
+      return "Big5"
+    end
+  end
+end

data/lib/rchardet/chardistribution.rb

@@ -0,0 +1,250 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Jeff Hodges
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  ENOUGH_DATA_THRESHOLD = 1024
+  SURE_YES = 0.99
+  SURE_NO = 0.01
+
+  class CharDistributionAnalysis
+    def initialize
+      @charToFreqOrder = nil # Mapping table to get frequency order from char order (get from GetOrder())
+      @tableSize = nil # Size of above table
+      @typicalDistributionRatio = nil # This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
+      reset()
+    end
+
+    def reset
+      # # """reset analyser, clear any state"""
+      @done = false # If this flag is set to constants.True, detection is done and conclusion has been made
+      @totalChars = 0 # Total characters encountered
+      @freqChars = 0 # The number of characters whose frequency order is less than 512
+    end
+
+    def feed(aStr, aCharLen)
+      # # """feed a character with known length"""
+      if aCharLen == 2
+        # we only care about 2-bytes character in our distribution analysis
+        order = get_order(aStr)
+      else
+        order = -1
+      end
+      if order >= 0
+        @totalChars += 1
+        # order is valid
+        if order < @tableSize
+          if 512 > @charToFreqOrder[order]
+            @freqChars += 1
+          end
+        end
+      end
+    end
+
+    def get_confidence
+      # """return confidence based on existing data"""
+      # if we didn't receive any character in our consideration range, return negative answer
+      if @totalChars <= 0
+        return SURE_NO
+      end
+
+      if @totalChars != @freqChars
+        r = @freqChars / ((@totalChars - @freqChars) * @typicalDistributionRatio)
+        if r < SURE_YES
+          return r
+        end
+      end
+
+      # normalize confidence (we don't want to be 100% sure)
+      return SURE_YES
+    end
+
+    def got_enough_data
+      # It is not necessary to receive all data to draw conclusion. For charset detection,
+      # certain amount of data is enough
+      return @totalChars > ENOUGH_DATA_THRESHOLD
+    end
+
+    def get_order(aStr)
+      # We do not handle characters based on the original encoding string, but 
+      # convert this encoding string to a number, here called order.
+      # This allows multiple encodings of a language to share one frequency table.
+      return -1
+    end
+  end
+
+  class EUCTWDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @charToFreqOrder = EUCTWCharToFreqOrder
+      @tableSize = EUCTW_TABLE_SIZE
+      @typicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for euc-TW encoding, we are interested 
+      #   first  byte range: 0xc4 -- 0xfe
+      #   second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0, 1] >= "\xC4"
+        bytes = aStr.bytes.to_a
+        return 94 * (bytes[0] - 0xC4) + bytes[1] - 0xA1
+      else
+        return -1
+      end
+    end
+
+    def get_confidence
+      if @freqChars <= MINIMUM_DATA_THRESHOLD
+        return SURE_NO
+      end
+
+      super
+    end
+  end
+
+  class EUCKRDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @charToFreqOrder = EUCKRCharToFreqOrder
+      @tableSize = EUCKR_TABLE_SIZE
+      @typicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for euc-KR encoding, we are interested 
+      #   first  byte range: 0xb0 -- 0xfe
+      #   second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0, 1] >= "\xB0"
+        bytes = aStr.bytes.to_a
+        return 94 * (bytes[0] - 0xB0) + bytes[1] - 0xA1
+      else
+        return -1
+      end
+    end
+  end
+
+  class GB18030DistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @charToFreqOrder = GB18030CharToFreqOrder
+      @tableSize = GB18030_TABLE_SIZE
+      @typicalDistributionRatio = GB18030_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for GB18030 encoding, we are interested 
+      #  first  byte range: 0xb0 -- 0xfe
+      #  second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if (aStr[0, 1] >= "\xB0") and (aStr[1, 1] >= "\xA1")
+        bytes = aStr.bytes.to_a
+        return 94 * (bytes[0] - 0xB0) + bytes[1] - 0xA1
+      else
+        return -1
+      end
+    end
+  end
+
+  class Big5DistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super
+      @charToFreqOrder = Big5CharToFreqOrder
+      @tableSize = BIG5_TABLE_SIZE
+      @typicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for big5 encoding, we are interested 
+      #   first  byte range: 0xa4 -- 0xfe
+      #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0, 1] >= "\xA4"
+        bytes = aStr.bytes.to_a
+        if aStr[1, 1] >= "\xA1"
+          return 157 * (bytes[0] - 0xA4) + bytes[1] - 0xA1 + 63
+        else
+          return 157 * (bytes[0] - 0xA4) + bytes[1] - 0x40
+        end
+      else
+        return -1
+      end
+    end
+  end
+
+  class SJISDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @charToFreqOrder = JISCharToFreqOrder
+      @tableSize = JIS_TABLE_SIZE
+      @typicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for sjis encoding, we are interested 
+      #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
+      #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
+      # no validation needed here. State machine has done that
+      bytes = aStr.bytes.to_a
+      if (aStr[0, 1] >= "\x81") and (aStr[0, 1] <= "\x9F")
+        order = 188 * (bytes[0] - 0x81)
+      elsif (aStr[0, 1] >= "\xE0") and (aStr[0, 1] <= "\xEF")
+        order = 188 * (bytes[0] - 0xE0 + 31)
+      else
+        return -1
+      end
+      order = order + bytes[1] - 0x40
+      if aStr[1, 1] > "\x7F"
+        order =- 1
+      end
+      return order
+    end
+  end
+
+  class EUCJPDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @charToFreqOrder = JISCharToFreqOrder
+      @tableSize = JIS_TABLE_SIZE
+      @typicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for euc-JP encoding, we are interested 
+      #   first  byte range: 0xa0 -- 0xfe
+      #   second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0, 1] >= "\xA0"
+        bytes = aStr.bytes.to_a
+        return 94 * (bytes[0] - 0xA1) + bytes[1] - 0xa1
+      else
+        return -1
+      end
+    end
+  end
+end

data/lib/rchardet/charsetgroupprober.rb

@@ -0,0 +1,110 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CharSetGroupProber < CharSetProber
+    attr_accessor :probers
+    def initialize
+      super
+      @activeNum = 0
+      @probers = []
+      @bestGuessProber = nil
+    end
+
+    def reset
+      super
+      @activeNum = 0
+
+      for prober in @probers
+        if prober
+          prober.reset()
+          prober.active = true
+          @activeNum += 1
+        end
+      end
+      @bestGuessProber = nil
+    end
+
+    def get_charset_name
+      if !@bestGuessProber
+        get_confidence()
+        if !@bestGuessProber
+          return nil
+        end
+      end
+      return @bestGuessProber.get_charset_name()
+    end
+
+    def feed(aBuf)
+      for prober in @probers
+        next unless prober
+        next unless prober.active
+        st = prober.feed(aBuf)
+        next unless st
+        if st == EFoundIt
+          @bestGuessProber = prober
+          return get_state()
+        elsif st == ENotMe
+          prober.active = false
+          @activeNum -= 1
+          if @activeNum <= 0
+            @state = ENotMe
+            return get_state()
+          end
+        end
+      end
+      return get_state()
+    end
+
+    def get_confidence()
+      st = get_state()
+      if st == EFoundIt
+        return 0.99
+      elsif st == ENotMe
+        return 0.01
+      end
+      bestConf = 0.0
+      @bestGuessProber = nil
+      for prober in @probers
+        next unless prober
+        unless prober.active
+          $stderr << "#{prober.get_charset_name()} not active\n" if $debug
+          next
+        end
+        cf = prober.get_confidence()
+        $stderr << "#{prober.get_charset_name} confidence = #{cf}\n" if $debug
+        if bestConf < cf
+          bestConf = cf
+          @bestGuessProber = prober
+        end
+      end
+      return 0.0 unless @bestGuessProber
+      return bestConf
+    end
+  end
+end

data/lib/rchardet/charsetprober.rb

@@ -0,0 +1,70 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CharSetProber
+    attr_accessor :active
+    def initialize
+    end
+
+    def reset
+      @state = EDetecting
+    end
+
+    def get_charset_name
+      return nil
+    end
+
+    def feed(aBuf)
+    end
+
+    def get_state
+      return @state
+    end
+
+    def get_confidence
+      return 0.0
+    end
+
+    def filter_high_bit_only(aBuf)
+      newBuf = aBuf.gsub(/([\x00-\x7F])+/, ' ')
+      return newBuf
+    end
+
+    def filter_without_english_letters(aBuf)
+      newBuf = aBuf.gsub(/([A-Za-z])+/,' ')
+      return newBuf
+    end
+
+    def filter_with_english_letters(aBuf)
+      # TODO
+      return aBuf
+    end
+  end
+end

data/lib/rchardet/codingstatemachine.rb

@@ -0,0 +1,67 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CodingStateMachine
+    attr_accessor :active
+
+    def initialize(sm)
+      @model = sm
+      @currentBytePos = 0
+      @currentCharLen = 0
+      reset()
+    end
+
+    def reset
+      @currentState = EStart
+    end
+
+    def next_state(c)
+      # for each byte we get its class
+      # if it is first byte, we also get byte length
+      b = c.bytes.first
+      byteCls = @model['classTable'][b]
+      if @currentState == EStart
+        @currentBytePos = 0
+        @currentCharLen = @model['charLenTable'][byteCls]
+      end
+      # from byte's class and stateTable, we get its next state
+      @currentState = @model['stateTable'][@currentState * @model['classFactor'] + byteCls]
+      @currentBytePos += 1
+      return @currentState
+    end
+
+    def get_current_charlen
+      return @currentCharLen
+    end
+
+    def get_coding_state_machine
+      return @model['name']
+    end
+  end
+end

data/lib/rchardet/constants.rb

@@ -0,0 +1,42 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  $debug = false
+
+  EDetecting = 0
+  EFoundIt = 1
+  ENotMe = 2
+
+  EStart = 0
+  EError = 1
+  EItsMe = 2
+
+  SHORTCUT_THRESHOLD = 0.95
+end

data/lib/rchardet/escprober.rb

@@ -0,0 +1,90 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class EscCharSetProber < CharSetProber
+    def initialize
+      super()
+      @codingSM = [ 
+                     CodingStateMachine.new(HZSMModel),
+                     CodingStateMachine.new(ISO2022CNSMModel),
+                     CodingStateMachine.new(ISO2022JPSMModel),
+                     CodingStateMachine.new(ISO2022KRSMModel)
+                    ]
+      reset()
+    end
+
+    def reset
+      super()
+      for codingSM in @codingSM
+        next if !codingSM
+        codingSM.active = true
+        codingSM.reset()
+      end
+      @activeSM = @codingSM.length
+      @detectedCharset = nil
+    end
+
+    def get_charset_name
+      return @detectedCharset
+    end
+
+    def get_confidence
+      if @detectedCharset
+        return 0.99
+      else
+        return 0.00
+      end
+    end
+
+    def feed(aBuf)
+      aBuf.each_byte do |b|
+        c = b.chr
+        for codingSM in @codingSM
+          next unless codingSM
+          next unless codingSM.active
+          codingState = codingSM.next_state(c)
+          if codingState == EError
+            codingSM.active = false
+            @activeSM -= 1
+            if @activeSM <= 0
+              @state = ENotMe
+              return get_state()
+            end
+          elsif codingState == EItsMe
+            @state = EFoundIt
+            @detectedCharset = codingSM.get_coding_state_machine()
+            return get_state()
+          end
+        end
+      end
+
+      return get_state()
+    end
+  end
+end