lg_pod_plugin 1.0.8 → 1.0.10

data/lib/rchardet/big5prober.rb

@@ -1,42 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-# 
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-# 
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  class Big5Prober < MultiByteCharSetProber 
-    def initialize
-      super
-      @codingSM = CodingStateMachine.new(Big5SMModel)
-      @distributionAnalyzer = Big5DistributionAnalysis.new()
-      reset()
-    end
-
-    def get_charset_name
-      return "Big5"
-    end
-  end
-end

data/lib/rchardet/chardistribution.rb

@@ -1,250 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-# 
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-# 
-# Contributor(s):
-#   Jeff Hodges
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  ENOUGH_DATA_THRESHOLD = 1024
-  SURE_YES = 0.99
-  SURE_NO = 0.01
-
-  class CharDistributionAnalysis
-    def initialize
-      @charToFreqOrder = nil # Mapping table to get frequency order from char order (get from GetOrder())
-      @tableSize = nil # Size of above table
-      @typicalDistributionRatio = nil # This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
-      reset()
-    end
-
-    def reset
-      # # """reset analyser, clear any state"""
-      @done = false # If this flag is set to constants.True, detection is done and conclusion has been made
-      @totalChars = 0 # Total characters encountered
-      @freqChars = 0 # The number of characters whose frequency order is less than 512
-    end
-
-    def feed(aStr, aCharLen)
-      # # """feed a character with known length"""
-      if aCharLen == 2
-        # we only care about 2-bytes character in our distribution analysis
-        order = get_order(aStr)
-      else
-        order = -1
-      end
-      if order >= 0
-        @totalChars += 1
-        # order is valid
-        if order < @tableSize
-          if 512 > @charToFreqOrder[order]
-            @freqChars += 1
-          end
-        end
-      end
-    end
-
-    def get_confidence
-      # """return confidence based on existing data"""
-      # if we didn't receive any character in our consideration range, return negative answer
-      if @totalChars <= 0
-        return SURE_NO
-      end
-
-      if @totalChars != @freqChars
-        r = @freqChars / ((@totalChars - @freqChars) * @typicalDistributionRatio)
-        if r < SURE_YES
-          return r
-        end
-      end
-
-      # normalize confidence (we don't want to be 100% sure)
-      return SURE_YES
-    end
-
-    def got_enough_data
-      # It is not necessary to receive all data to draw conclusion. For charset detection,
-      # certain amount of data is enough
-      return @totalChars > ENOUGH_DATA_THRESHOLD
-    end
-
-    def get_order(aStr)
-      # We do not handle characters based on the original encoding string, but 
-      # convert this encoding string to a number, here called order.
-      # This allows multiple encodings of a language to share one frequency table.
-      return -1
-    end
-  end
-
-  class EUCTWDistributionAnalysis < CharDistributionAnalysis
-    def initialize
-      super()
-      @charToFreqOrder = EUCTWCharToFreqOrder
-      @tableSize = EUCTW_TABLE_SIZE
-      @typicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
-    end
-
-    def get_order(aStr)
-      # for euc-TW encoding, we are interested 
-      #   first  byte range: 0xc4 -- 0xfe
-      #   second byte range: 0xa1 -- 0xfe
-      # no validation needed here. State machine has done that
-      if aStr[0, 1] >= "\xC4"
-        bytes = aStr.bytes.to_a
-        return 94 * (bytes[0] - 0xC4) + bytes[1] - 0xA1
-      else
-        return -1
-      end
-    end
-
-    def get_confidence
-      if @freqChars <= MINIMUM_DATA_THRESHOLD
-        return SURE_NO
-      end
-
-      super
-    end
-  end
-
-  class EUCKRDistributionAnalysis < CharDistributionAnalysis
-    def initialize
-      super()
-      @charToFreqOrder = EUCKRCharToFreqOrder
-      @tableSize = EUCKR_TABLE_SIZE
-      @typicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
-    end
-
-    def get_order(aStr)
-      # for euc-KR encoding, we are interested 
-      #   first  byte range: 0xb0 -- 0xfe
-      #   second byte range: 0xa1 -- 0xfe
-      # no validation needed here. State machine has done that
-      if aStr[0, 1] >= "\xB0"
-        bytes = aStr.bytes.to_a
-        return 94 * (bytes[0] - 0xB0) + bytes[1] - 0xA1
-      else
-        return -1
-      end
-    end
-  end
-
-  class GB18030DistributionAnalysis < CharDistributionAnalysis
-    def initialize
-      super()
-      @charToFreqOrder = GB18030CharToFreqOrder
-      @tableSize = GB18030_TABLE_SIZE
-      @typicalDistributionRatio = GB18030_TYPICAL_DISTRIBUTION_RATIO
-    end
-
-    def get_order(aStr)
-      # for GB18030 encoding, we are interested 
-      #  first  byte range: 0xb0 -- 0xfe
-      #  second byte range: 0xa1 -- 0xfe
-      # no validation needed here. State machine has done that
-      if (aStr[0, 1] >= "\xB0") and (aStr[1, 1] >= "\xA1")
-        bytes = aStr.bytes.to_a
-        return 94 * (bytes[0] - 0xB0) + bytes[1] - 0xA1
-      else
-        return -1
-      end
-    end
-  end
-
-  class Big5DistributionAnalysis < CharDistributionAnalysis
-    def initialize
-      super
-      @charToFreqOrder = Big5CharToFreqOrder
-      @tableSize = BIG5_TABLE_SIZE
-      @typicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
-    end
-
-    def get_order(aStr)
-      # for big5 encoding, we are interested 
-      #   first  byte range: 0xa4 -- 0xfe
-      #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
-      # no validation needed here. State machine has done that
-      if aStr[0, 1] >= "\xA4"
-        bytes = aStr.bytes.to_a
-        if aStr[1, 1] >= "\xA1"
-          return 157 * (bytes[0] - 0xA4) + bytes[1] - 0xA1 + 63
-        else
-          return 157 * (bytes[0] - 0xA4) + bytes[1] - 0x40
-        end
-      else
-        return -1
-      end
-    end
-  end
-
-  class SJISDistributionAnalysis < CharDistributionAnalysis
-    def initialize
-      super()
-      @charToFreqOrder = JISCharToFreqOrder
-      @tableSize = JIS_TABLE_SIZE
-      @typicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
-    end
-
-    def get_order(aStr)
-      # for sjis encoding, we are interested 
-      #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
-      #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
-      # no validation needed here. State machine has done that
-      bytes = aStr.bytes.to_a
-      if (aStr[0, 1] >= "\x81") and (aStr[0, 1] <= "\x9F")
-        order = 188 * (bytes[0] - 0x81)
-      elsif (aStr[0, 1] >= "\xE0") and (aStr[0, 1] <= "\xEF")
-        order = 188 * (bytes[0] - 0xE0 + 31)
-      else
-        return -1
-      end
-      order = order + bytes[1] - 0x40
-      if aStr[1, 1] > "\x7F"
-        order =- 1
-      end
-      return order
-    end
-  end
-
-  class EUCJPDistributionAnalysis < CharDistributionAnalysis
-    def initialize
-      super()
-      @charToFreqOrder = JISCharToFreqOrder
-      @tableSize = JIS_TABLE_SIZE
-      @typicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
-    end
-
-    def get_order(aStr)
-      # for euc-JP encoding, we are interested 
-      #   first  byte range: 0xa0 -- 0xfe
-      #   second byte range: 0xa1 -- 0xfe
-      # no validation needed here. State machine has done that
-      if aStr[0, 1] >= "\xA0"
-        bytes = aStr.bytes.to_a
-        return 94 * (bytes[0] - 0xA1) + bytes[1] - 0xa1
-      else
-        return -1
-      end
-    end
-  end
-end

data/lib/rchardet/charsetgroupprober.rb

@@ -1,110 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Communicator client code.
-# 
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-# 
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  class CharSetGroupProber < CharSetProber
-    attr_accessor :probers
-    def initialize
-      super
-      @activeNum = 0
-      @probers = []
-      @bestGuessProber = nil
-    end
-
-    def reset
-      super
-      @activeNum = 0
-
-      for prober in @probers
-        if prober
-          prober.reset()
-          prober.active = true
-          @activeNum += 1
-        end
-      end
-      @bestGuessProber = nil
-    end
-
-    def get_charset_name
-      if !@bestGuessProber
-        get_confidence()
-        if !@bestGuessProber
-          return nil
-        end
-      end
-      return @bestGuessProber.get_charset_name()
-    end
-
-    def feed(aBuf)
-      for prober in @probers
-        next unless prober
-        next unless prober.active
-        st = prober.feed(aBuf)
-        next unless st
-        if st == EFoundIt
-          @bestGuessProber = prober
-          return get_state()
-        elsif st == ENotMe
-          prober.active = false
-          @activeNum -= 1
-          if @activeNum <= 0
-            @state = ENotMe
-            return get_state()
-          end
-        end
-      end
-      return get_state()
-    end
-
-    def get_confidence()
-      st = get_state()
-      if st == EFoundIt
-        return 0.99
-      elsif st == ENotMe
-        return 0.01
-      end
-      bestConf = 0.0
-      @bestGuessProber = nil
-      for prober in @probers
-        next unless prober
-        unless prober.active
-          $stderr << "#{prober.get_charset_name()} not active\n" if $debug
-          next
-        end
-        cf = prober.get_confidence()
-        $stderr << "#{prober.get_charset_name} confidence = #{cf}\n" if $debug
-        if bestConf < cf
-          bestConf = cf
-          @bestGuessProber = prober
-        end
-      end
-      return 0.0 unless @bestGuessProber
-      return bestConf
-    end
-  end
-end

data/lib/rchardet/charsetprober.rb

@@ -1,70 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-# 
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-# 
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  class CharSetProber
-    attr_accessor :active
-    def initialize
-    end
-
-    def reset
-      @state = EDetecting
-    end
-
-    def get_charset_name
-      return nil
-    end
-
-    def feed(aBuf)
-    end
-
-    def get_state
-      return @state
-    end
-
-    def get_confidence
-      return 0.0
-    end
-
-    def filter_high_bit_only(aBuf)
-      newBuf = aBuf.gsub(/([\x00-\x7F])+/, ' ')
-      return newBuf
-    end
-
-    def filter_without_english_letters(aBuf)
-      newBuf = aBuf.gsub(/([A-Za-z])+/,' ')
-      return newBuf
-    end
-
-    def filter_with_english_letters(aBuf)
-      # TODO
-      return aBuf
-    end
-  end
-end

data/lib/rchardet/codingstatemachine.rb

@@ -1,67 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  class CodingStateMachine
-    attr_accessor :active
-
-    def initialize(sm)
-      @model = sm
-      @currentBytePos = 0
-      @currentCharLen = 0
-      reset()
-    end
-
-    def reset
-      @currentState = EStart
-    end
-
-    def next_state(c)
-      # for each byte we get its class
-      # if it is first byte, we also get byte length
-      b = c.bytes.first
-      byteCls = @model['classTable'][b]
-      if @currentState == EStart
-        @currentBytePos = 0
-        @currentCharLen = @model['charLenTable'][byteCls]
-      end
-      # from byte's class and stateTable, we get its next state
-      @currentState = @model['stateTable'][@currentState * @model['classFactor'] + byteCls]
-      @currentBytePos += 1
-      return @currentState
-    end
-
-    def get_current_charlen
-      return @currentCharLen
-    end
-
-    def get_coding_state_machine
-      return @model['name']
-    end
-  end
-end

data/lib/rchardet/constants.rb

@@ -1,42 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  $debug = false
-
-  EDetecting = 0
-  EFoundIt = 1
-  ENotMe = 2
-
-  EStart = 0
-  EError = 1
-  EItsMe = 2
-
-  SHORTCUT_THRESHOLD = 0.95
-end

data/lib/rchardet/escprober.rb

@@ -1,90 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-module CharDet
-  class EscCharSetProber < CharSetProber
-    def initialize
-      super()
-      @codingSM = [ 
-                     CodingStateMachine.new(HZSMModel),
-                     CodingStateMachine.new(ISO2022CNSMModel),
-                     CodingStateMachine.new(ISO2022JPSMModel),
-                     CodingStateMachine.new(ISO2022KRSMModel)
-                    ]
-      reset()
-    end
-
-    def reset
-      super()
-      for codingSM in @codingSM
-        next if !codingSM
-        codingSM.active = true
-        codingSM.reset()
-      end
-      @activeSM = @codingSM.length
-      @detectedCharset = nil
-    end
-
-    def get_charset_name
-      return @detectedCharset
-    end
-
-    def get_confidence
-      if @detectedCharset
-        return 0.99
-      else
-        return 0.00
-      end
-    end
-
-    def feed(aBuf)
-      aBuf.each_byte do |b|
-        c = b.chr
-        for codingSM in @codingSM
-          next unless codingSM
-          next unless codingSM.active
-          codingState = codingSM.next_state(c)
-          if codingState == EError
-            codingSM.active = false
-            @activeSM -= 1
-            if @activeSM <= 0
-              @state = ENotMe
-              return get_state()
-            end
-          elsif codingState == EItsMe
-            @state = EFoundIt
-            @detectedCharset = codingSM.get_coding_state_machine()
-            return get_state()
-          end
-        end
-      end
-
-      return get_state()
-    end
-  end
-end