tmail_es 1.2.7.2

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/big5prober.rb

@@ -0,0 +1,42 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class Big5Prober < MultiByteCharSetProber 
+    def initialize
+      super
+      @_mCodingSM = CodingStateMachine.new(Big5SMModel)
+      @_mDistributionAnalyzer = Big5DistributionAnalysis.new()
+      reset()
+    end
+
+    def get_charset_name
+      return "Big5"
+    end
+  end
+end

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/chardistribution.rb

@@ -0,0 +1,238 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s)
+
+#   Jeff Hodges
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  ENOUGH_DATA_THRESHOLD = 1024
+  SURE_YES = 0.99
+  SURE_NO = 0.01
+
+  class CharDistributionAnalysis
+    def initialize
+      @_mCharToFreqOrder = nil # Mapping table to get frequency order from char order (get from GetOrder())
+      @_mTableSize = nil # Size of above table
+      @_mTypicalDistributionRatio = nil # This is a constant value which varies from language to language, used in calculating confidence.  See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
+      reset()
+    end
+
+    def reset
+      # # """reset analyser, clear any state"""
+      @_mDone = false # If this flag is set to constants.True, detection is done and conclusion has been made
+      @_mTotalChars = 0 # Total characters encountered
+      @_mFreqChars = 0 # The number of characters whose frequency order is less than 512
+    end
+
+    def feed(aStr, aCharLen)
+      # # """feed a character with known length"""
+      if aCharLen == 2
+        # we only care about 2-bytes character in our distribution analysis
+        order = get_order(aStr)
+      else
+        order = -1
+      end
+      if order >= 0
+        @_mTotalChars += 1
+        # order is valid
+        if order < @_mTableSize
+          if 512 > @_mCharToFreqOrder[order]
+            @_mFreqChars += 1
+          end
+        end
+      end
+    end
+
+    def get_confidence
+      # """return confidence based on existing data"""
+      # if we didn't receive any character in our consideration range, return negative answer
+      if @_mTotalChars <= 0
+        return SURE_NO
+      end
+
+      if @_mTotalChars != @_mFreqChars
+        r = @_mFreqChars / ((@_mTotalChars - @_mFreqChars) * @_mTypicalDistributionRatio)
+        if r < SURE_YES
+          return r
+        end
+      end
+
+      # normalize confidence (we don't want to be 100% sure)
+      return SURE_YES
+    end
+
+    def got_enough_data
+      # It is not necessary to receive all data to draw conclusion. For charset detection,
+      # certain amount of data is enough
+      return @_mTotalChars > ENOUGH_DATA_THRESHOLD
+    end
+
+    def get_order(aStr)
+      # We do not handle characters based on the original encoding string, but 
+      # convert this encoding string to a number, here called order.
+      # This allows multiple encodings of a language to share one frequency table.
+      return -1
+    end
+  end
+
+  class EUCTWDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @_mCharToFreqOrder = EUCTWCharToFreqOrder
+      @_mTableSize = EUCTW_TABLE_SIZE
+      @_mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for euc-TW encoding, we are interested 
+      #   first  byte range: 0xc4 -- 0xfe
+      #   second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0..0] >= "\xC4"
+        return 94 * (aStr[0] - 0xC4) + aStr[1] - 0xA1
+      else
+        return -1
+      end
+    end
+  end
+
+  class EUCKRDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @_mCharToFreqOrder = EUCKRCharToFreqOrder
+      @_mTableSize = EUCKR_TABLE_SIZE
+      @_mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for euc-KR encoding, we are interested 
+      #   first  byte range: 0xb0 -- 0xfe
+      #   second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0..0] >= "\xB0"
+        return 94 * (aStr[0] - 0xB0) + aStr[1] - 0xA1
+      else
+        return -1
+      end
+    end
+  end
+
+  class GB2312DistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @_mCharToFreqOrder = GB2312CharToFreqOrder
+      @_mTableSize = GB2312_TABLE_SIZE
+      @_mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for GB2312 encoding, we are interested 
+      #  first  byte range: 0xb0 -- 0xfe
+      #  second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if (aStr[0..0] >= "\xB0") and (aStr[1..1] >= "\xA1")
+        return 94 * (aStr[0] - 0xB0) + aStr[1] - 0xA1
+      else
+        return -1
+      end
+    end
+  end
+
+  class Big5DistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super
+      @_mCharToFreqOrder = Big5CharToFreqOrder
+      @_mTableSize = BIG5_TABLE_SIZE
+      @_mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for big5 encoding, we are interested 
+      #   first  byte range: 0xa4 -- 0xfe
+      #   second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0..0] >= "\xA4"
+        if aStr[1..1] >= "\xA1"
+          return 157 * (aStr[0] - 0xA4) + aStr[1] - 0xA1 + 63
+        else
+          return 157 * (aStr[0] - 0xA4) + aStr[1] - 0x40
+        end
+      else
+        return -1
+      end
+    end
+  end
+
+  class SJISDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @_mCharToFreqOrder = JISCharToFreqOrder
+      @_mTableSize = JIS_TABLE_SIZE
+      @_mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for sjis encoding, we are interested 
+      #   first  byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
+      #   second byte range: 0x40 -- 0x7e,  0x81 -- oxfe
+      # no validation needed here. State machine has done that
+      aStr = aStr[0..1].join if aStr.class == Array
+      if (aStr[0..0] >= "\x81") and (aStr[0..0] <= "\x9F")
+        order = 188 * (aStr[0] - 0x81)
+      elsif (aStr[0..0] >= "\xE0") and (aStr[0..0] <= "\xEF")
+        order = 188 * (aStr[0] - 0xE0 + 31)
+      else
+        return -1
+      end
+      order = order + aStr[1] - 0x40
+      if aStr[1..1] > "\x7F"
+        order =- 1
+      end
+      return order
+    end
+  end
+
+  class EUCJPDistributionAnalysis < CharDistributionAnalysis
+    def initialize
+      super()
+      @_mCharToFreqOrder = JISCharToFreqOrder
+      @_mTableSize = JIS_TABLE_SIZE
+      @_mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
+    end
+
+    def get_order(aStr)
+      # for euc-JP encoding, we are interested 
+      #   first  byte range: 0xa0 -- 0xfe
+      #   second byte range: 0xa1 -- 0xfe
+      # no validation needed here. State machine has done that
+      if aStr[0..0] >= "\xA0"
+        return 94 * (aStr[0] - 0xA1) + aStr[1] - 0xa1
+      else
+        return -1
+      end
+    end
+  end
+end

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/charsetgroupprober.rb

@@ -0,0 +1,112 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Communicator client code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s)
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CharSetGroupProber < CharSetProber
+    attr_accessor :_mProbers
+    def initialize
+      super
+      @_mActiveNum = 0
+      @_mProbers = []
+      @_mBestGuessProber = nil
+    end
+
+    def reset
+      super
+      @_mActiveNum = 0
+
+      for prober in @_mProbers
+        if prober
+          prober.reset()
+          prober.active = true
+          @_mActiveNum += 1
+        end
+      end
+      @_mBestGuessProber = nil
+    end
+
+    def get_charset_name
+      if not @_mBestGuessProber
+        get_confidence()
+        return nil unless @_mBestGuessProber
+        #                self._mBestGuessProber = self._mProbers[0]
+      end
+      return @_mBestGuessProber.get_charset_name()
+    end
+
+    def feed(aBuf)
+      for prober in @_mProbers
+        next unless prober
+        next unless prober.active
+        st = prober.feed(aBuf)
+        next unless st
+        if st == EFoundIt
+          @_mBestGuessProber = prober
+          return get_state()
+        elsif st == ENotMe
+          prober.active = false
+          @_mActiveNum -= 1
+          if @_mActiveNum <= 0
+            @_mState = ENotMe
+            return get_state()
+          end
+        end
+      end
+      return get_state()
+    end
+
+    def get_confidence()
+      st = get_state()
+      if st == EFoundIt
+        return 0.99
+      elsif st == ENotMe
+        return 0.01
+      end
+      bestConf = 0.0
+      @_mBestGuessProber = nil
+      for prober in @_mProbers
+        next unless prober
+        unless prober.active
+          $stderr << "#{prober.get_charset_name()} not active\n" if $debug
+          next
+        end
+        cf = prober.get_confidence()
+        $stderr << "#{prober.get_charset_name} confidence = #{cf}\n" if $debug
+        if bestConf < cf
+          bestConf = cf
+          @_mBestGuessProber = prober
+        end
+      end
+      return 0.0 unless @_mBestGuessProber
+      return bestConf
+      #        else
+      #            self._mBestGuessProber = self._mProbers[0]
+      #            return self._mBestGuessProber.get_confidence()
+    end
+  end
+end

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/charsetprober.rb

@@ -0,0 +1,75 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+# 
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+# 
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CharSetProber
+    attr_accessor :active
+    def initialize
+    end
+
+    def reset
+      @_mState = EDetecting
+    end
+
+    def get_charset_name
+      return nil
+    end
+
+    def feed(aBuf)
+    end
+
+    def get_state
+      return @_mState
+    end
+
+    def get_confidence
+      return 0.0
+    end
+
+    def filter_high_bit_only(aBuf)
+      # DO NOT USE `gsub!`
+      # It will remove all characters from the buffer that is later used by
+      # other probers.  This is because gsub! removes data from the instance variable
+      # that will be passed to later probers, while gsub makes a new instance variable
+      # that will not. 
+      newBuf = aBuf.gsub(/([\x00-\x7F])+/, ' ')
+      return newBuf
+    end
+
+    def filter_without_english_letters(aBuf)
+      newBuf = aBuf.gsub(/([A-Za-z])+/,' ')
+      return newBuf
+    end
+
+    def filter_with_english_letters(aBuf)
+      # TODO
+      return aBuf
+    end
+  end
+end

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/codingstatemachine.rb

@@ -0,0 +1,64 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class CodingStateMachine
+    def initialize(sm)
+      @_mModel = sm
+      @_mCurrentBytePos = 0
+      @_mCurrentCharLen = 0
+      reset()
+    end
+
+    def reset
+      @_mCurrentState = EStart
+    end
+
+    def next_state(c)
+      # for each byte we get its class
+      # if it is first byte, we also get byte length
+      byteCls = @_mModel['classTable'][c[0]]
+      if @_mCurrentState == EStart
+        @_mCurrentBytePos = 0
+        @_mCurrentCharLen = @_mModel['charLenTable'][byteCls]
+      end
+      # from byte's class and stateTable, we get its next state
+      @_mCurrentState = @_mModel['stateTable'][@_mCurrentState * @_mModel['classFactor'] + byteCls]
+      @_mCurrentBytePos += 1
+      return @_mCurrentState
+    end
+
+    def get_current_charlen
+      return @_mCurrentCharLen
+    end
+
+    def get_coding_state_machine
+      return @_mModel['name']
+    end
+  end
+end

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/constants.rb

@@ -0,0 +1,42 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  $debug = false
+
+  EDetecting = 0
+  EFoundIt = 1
+  ENotMe = 2
+
+  EStart = 0
+  EError = 1
+  EItsMe = 2
+
+  SHORTCUT_THRESHOLD = 0.95
+end

data/lib/tmail/vendor/rchardet-1.3/lib/rchardet/escprober.rb

@@ -0,0 +1,89 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class EscCharSetProber < CharSetProber
+    def initialize
+      super()
+      @_mCodingSM = [ CodingStateMachine.new(HZSMModel),
+                      CodingStateMachine.new(ISO2022CNSMModel),
+                      CodingStateMachine.new(ISO2022JPSMModel),
+                      CodingStateMachine.new(ISO2022KRSMModel)  ]
+      reset()
+    end
+
+    def reset
+      super()
+      for codingSM in @_mCodingSM
+        next if not codingSM
+        codingSM.active = true
+        codingSM.reset()
+      end
+      @_mActiveSM = @_mCodingSM.length
+      @_mDetectedCharset = nil
+    end
+
+    def get_charset_name
+      return @_mDetectedCharset
+    end
+
+    def get_confidence
+      if @_mDetectedCharset
+        return 0.99
+      else
+        return 0.00
+      end
+    end
+
+    def feed(aBuf)
+      aBuf.each_byte do |b|
+        c = b.chr
+        for codingSM in @_mCodingSM
+          next unless codingSM
+          next unless codingSM.active
+          codingState = codingSM.next_state(c)
+          if codingState == EError
+            codingSM.active = false
+            @_mActiveSM -= 1
+            if @_mActiveSM <= 0
+              @_mState = ENotMe
+              return get_state()
+            end
+          elsif codingState == EItsMe
+            @_mState = EFoundIt
+            @_mDetectedCharset = codingSM.get_coding_state_machine()
+            return get_state()
+          end
+        end
+      end
+      return get_state()
+
+    end
+
+  end
+end