radiant 0.9.0.rc2 → 0.9.1

data/vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/sbcharsetprober.rb

@@ -0,0 +1,124 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  SAMPLE_SIZE = 64
+  SB_ENOUGH_REL_THRESHOLD = 1024
+  POSITIVE_SHORTCUT_THRESHOLD = 0.95
+  NEGATIVE_SHORTCUT_THRESHOLD = 0.05
+  SYMBOL_CAT_ORDER = 250
+  NUMBER_OF_SEQ_CAT = 4
+  POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
+  #NEGATIVE_CAT = 0
+
+  class SingleByteCharSetProber < CharSetProber
+    def initialize(model, reversed=false, nameProber=nil)
+      super()
+      @_mModel = model
+      @_mReversed = reversed # TRUE if we need to reverse every pair in the model lookup
+      @_mNameProber = nameProber # Optional auxiliary prober for name decision
+      reset()
+    end
+
+    def reset
+      super()
+      @_mLastOrder = 255 # char order of last character
+      @_mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
+      @_mTotalSeqs = 0
+      @_mTotalChar = 0
+      @_mFreqChar = 0 # characters that fall in our sampling range
+    end
+
+    def get_charset_name
+      if @_mNameProber
+        return @_mNameProber.get_charset_name()
+      else
+        return @_mModel['charsetName']
+      end
+    end
+
+    def feed(aBuf)
+      if not @_mModel['keepEnglishLetter']
+        aBuf = filter_without_english_letters(aBuf)
+      end
+      aLen = aBuf.length
+      if not aLen
+        return get_state()
+      end
+      aBuf.each_byte do |b|
+        c = b.chr
+        order = @_mModel['charToOrderMap'][c[0]]
+        if order < SYMBOL_CAT_ORDER
+          @_mTotalChar += 1
+        end
+        if order < SAMPLE_SIZE
+          @_mFreqChar += 1
+          if @_mLastOrder < SAMPLE_SIZE
+            @_mTotalSeqs += 1
+            if not @_mReversed
+              @_mSeqCounters[@_mModel['precedenceMatrix'][(@_mLastOrder * SAMPLE_SIZE) + order]] += 1
+            else # reverse the order of the letters in the lookup
+              @_mSeqCounters[@_mModel['precedenceMatrix'][(order * SAMPLE_SIZE) + @_mLastOrder]] += 1
+            end
+          end
+        end
+        @_mLastOrder = order
+      end
+
+      if get_state() == EDetecting
+        if @_mTotalSeqs > SB_ENOUGH_REL_THRESHOLD
+          cf = get_confidence()
+          if cf > POSITIVE_SHORTCUT_THRESHOLD
+            $stderr << "#{@_mModel['charsetName']} confidence = #{cf}, we have a winner\n" if $debug
+            @_mState = EFoundIt
+          elsif cf < NEGATIVE_SHORTCUT_THRESHOLD
+            $stderr << "#{@_mModel['charsetName']} confidence = #{cf}, below negative shortcut threshold #{NEGATIVE_SHORTCUT_THRESHOLD}\n" if $debug
+            @_mState = ENotMe
+          end
+        end
+      end
+
+      return get_state()
+    end
+
+    def get_confidence
+      r = 0.01
+      if @_mTotalSeqs > 0
+        #            print self._mSeqCounters[POSITIVE_CAT], self._mTotalSeqs, self._mModel['mTypicalPositiveRatio']
+        r = (1.0 * @_mSeqCounters[POSITIVE_CAT]) / @_mTotalSeqs / @_mModel['mTypicalPositiveRatio']
+        #            print r, self._mFreqChar, self._mTotalChar
+        r = r * @_mFreqChar / @_mTotalChar
+        if r >= 1.0
+          r = 0.99
+        end
+      end
+      return r
+    end
+  end
+end

data/vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/sbcsgroupprober.rb

@@ -0,0 +1,56 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class SBCSGroupProber < CharSetGroupProber
+    def initialize
+      super
+      @_mProbers = [ SingleByteCharSetProber.new(Win1251CyrillicModel),
+                     SingleByteCharSetProber.new(Koi8rModel),
+                     SingleByteCharSetProber.new(Latin5CyrillicModel),
+                     SingleByteCharSetProber.new(MacCyrillicModel),
+                     SingleByteCharSetProber.new(Ibm866Model),
+                     SingleByteCharSetProber.new(Ibm855Model),
+                     SingleByteCharSetProber.new(Latin7GreekModel),
+                     SingleByteCharSetProber.new(Win1253GreekModel),
+                     SingleByteCharSetProber.new(Latin5BulgarianModel),
+                     SingleByteCharSetProber.new(Win1251BulgarianModel),
+                     SingleByteCharSetProber.new(Latin2HungarianModel),
+                     SingleByteCharSetProber.new(Win1250HungarianModel),
+                     SingleByteCharSetProber.new(TIS620ThaiModel) ]
+      hebrewProber = HebrewProber.new()
+      logicalHebrewProber = SingleByteCharSetProber.new(Win1255HebrewModel, false, hebrewProber)
+      visualHebrewProber = SingleByteCharSetProber.new(Win1255HebrewModel, true, hebrewProber)
+      hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber)
+      @_mProbers += [hebrewProber, logicalHebrewProber, visualHebrewProber]
+
+      reset()
+    end
+  end
+end

data/vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/sjisprober.rb

@@ -0,0 +1,88 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  class SJISProber < MultiByteCharSetProber
+    def initialize
+      super()
+      @_mCodingSM = CodingStateMachine.new(SJISSMModel)
+      @_mDistributionAnalyzer = SJISDistributionAnalysis.new()
+      @_mContextAnalyzer = SJISContextAnalysis.new()
+      reset()
+    end
+
+    def reset
+      super()
+      @_mContextAnalyzer.reset()
+    end
+
+    def get_charset_name
+      return "SHIFT_JIS"
+    end
+
+    def feed(aBuf)
+      aLen = aBuf.length
+      for i in (0...aLen)
+        codingState = @_mCodingSM.next_state(aBuf[i..i])
+        if codingState == EError
+          $stderr << "#{get_charset_name} prober hit error at byte #{i}\n" if $debug
+          @_mState = ENotMe
+          break
+        elsif codingState == EItsMe
+          @_mState = EFoundIt
+          break
+        elsif codingState == EStart
+          charLen = @_mCodingSM.get_current_charlen()
+          if i == 0
+            @_mLastChar[1] = aBuf[0..0]
+            @_mContextAnalyzer.feed(@_mLastChar[2 - charLen..-1], charLen)
+            @_mDistributionAnalyzer.feed(@_mLastChar, charLen)
+          else
+            @_mContextAnalyzer.feed(aBuf[i + 1 - charLen ... i + 3 - charLen], charLen)
+            @_mDistributionAnalyzer.feed(aBuf[i - 1 ... i + 1], charLen)
+          end
+        end
+      end
+
+      @_mLastChar[0] = aBuf[aLen - 1.. aLen-1]
+
+      if get_state() == EDetecting
+        if @_mContextAnalyzer.got_enough_data() and (get_confidence() > SHORTCUT_THRESHOLD)
+          @_mState = EFoundIt
+        end
+      end
+
+      return get_state()
+    end
+
+    def get_confidence
+      l = [@_mContextAnalyzer.get_confidence(), @_mDistributionAnalyzer.get_confidence()]
+      return l.max
+    end
+  end
+end

data/vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/universaldetector.rb

@@ -0,0 +1,168 @@
+# encoding: us-ascii
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  MINIMUM_THRESHOLD = 0.20
+  EPureAscii = 0
+  EEscAscii = 1
+  EHighbyte = 2
+
+  class UniversalDetector
+    attr_accessor :result
+    def initialize
+      @_highBitDetector = /[\x80-\xFF]/
+      @_escDetector = /(\033|\~\{)/
+      @_mEscCharSetProber = nil
+      @_mCharSetProbers = []
+      reset()
+    end
+
+    def reset
+      @result = {'encoding' => nil, 'confidence' => 0.0}
+      @done = false
+      @_mStart = true
+      @_mGotData = false
+      @_mInputState = EPureAscii
+      @_mLastChar = ''
+      if @_mEscCharSetProber
+        @_mEscCharSetProber.reset()
+      end
+      for prober in @_mCharSetProbers
+        prober.reset()
+      end
+    end
+
+    def feed(aBuf)
+      return if @done
+
+      aLen = aBuf.length
+      return if not aLen
+
+      if not @_mGotData
+        # If the data starts with BOM, we know it is UTF
+        if aBuf[0...3] == "\xEF\xBB\xBF"
+          # EF BB BF  UTF-8 with BOM
+          @result = {'encoding' => "UTF-8", 'confidence' => 1.0}
+        elsif aBuf[0...4] == "\xFF\xFE\x00\x00"
+          # FF FE 00 00  UTF-32, little-endian BOM
+          @result = {'encoding' => "UTF-32LE", 'confidence' => 1.0}
+        elsif aBuf[0...4] == "\x00\x00\xFE\xFF"
+          # 00 00 FE FF  UTF-32, big-endian BOM
+          @result = {'encoding' => "UTF-32BE", 'confidence' => 1.0}
+        elsif aBuf[0...4] == "\xFE\xFF\x00\x00"
+          # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
+          @result = {'encoding' => "X-ISO-10646-UCS-4-3412", 'confidence' => 1.0}
+        elsif aBuf[0...4] == "\x00\x00\xFF\xFE"
+          # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
+          @result = {'encoding' =>  "X-ISO-10646-UCS-4-2143", 'confidence' =>  1.0}
+        elsif aBuf[0...2] == "\xFF\xFE"
+          # FF FE  UTF-16, little endian BOM
+          @result = {'encoding' =>  "UTF-16LE", 'confidence' =>  1.0}
+        elsif aBuf[0...2] == "\xFE\xFF"
+          # FE FF  UTF-16, big endian BOM
+          @result = {'encoding' =>  "UTF-16BE", 'confidence' =>  1.0}
+        end
+      end
+
+      @_mGotData = true
+      if @result['encoding'] and (@result['confidence'] > 0.0)
+        @done = true
+        return
+      end
+
+      if @_mInputState == EPureAscii
+        if @_highBitDetector =~ (aBuf)
+          @_mInputState = EHighbyte
+        elsif (@_mInputState == EPureAscii) and @_escDetector =~ (@_mLastChar + aBuf)
+          @_mInputState = EEscAscii
+        end
+      end
+
+      @_mLastChar = aBuf[-1..-1]
+      if @_mInputState == EEscAscii
+        if not @_mEscCharSetProber
+          @_mEscCharSetProber = EscCharSetProber.new()
+        end
+        if @_mEscCharSetProber.feed(aBuf) == EFoundIt
+          @result = {'encoding' =>  self._mEscCharSetProber.get_charset_name(),
+                     'confidence' =>  @_mEscCharSetProber.get_confidence()
+          }
+          @done = true
+        end
+      elsif @_mInputState == EHighbyte
+        if not @_mCharSetProbers or @_mCharSetProbers.empty?
+          @_mCharSetProbers = [MBCSGroupProber.new(), SBCSGroupProber.new(), Latin1Prober.new()]
+        end
+        for prober in @_mCharSetProbers
+          if prober.feed(aBuf) == EFoundIt
+            @result = {'encoding' =>  prober.get_charset_name(),
+                       'confidence' =>  prober.get_confidence()}
+            @done = true
+            break
+          end
+        end
+      end
+
+    end
+
+    def close
+      return if @done
+      if not @_mGotData
+        $stderr << "no data received!\n" if $debug
+        return
+      end
+      @done = true
+
+      if @_mInputState == EPureAscii
+        @result = {'encoding' => 'ascii', 'confidence' => 1.0}
+        return @result
+      end
+
+      if @_mInputState == EHighbyte
+        confidences = {}
+        @_mCharSetProbers.each{ |prober| confidences[prober] = prober.get_confidence }
+        maxProber = @_mCharSetProbers.max{ |a,b| confidences[a] <=> confidences[b] }
+        if maxProber and maxProber.get_confidence > MINIMUM_THRESHOLD
+          @result = {'encoding' =>  maxProber.get_charset_name(),
+                     'confidence' =>  maxProber.get_confidence()}
+          return @result
+        end
+      end
+
+      if $debug
+        $stderr << "no probers hit minimum threshhold\n" if $debug
+        for prober in @_mCharSetProbers[0]._mProbers
+          next if not prober
+          $stderr << "#{prober.get_charset_name} confidence = #{prober.get_confidence}\n" if $debug
+        end
+      end
+    end
+  end
+end

data/vendor/rails/actionmailer/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/utf8prober.rb

@@ -0,0 +1,87 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Jeff Hodges - port to Ruby
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+module CharDet
+  ONE_CHAR_PROB = 0.5
+
+  class UTF8Prober < CharSetProber
+    def initialize
+      super()
+      @_mCodingSM = CodingStateMachine.new(UTF8SMModel)
+      reset()
+    end
+
+    def reset
+      super()
+      @_mCodingSM.reset()
+      @_mNumOfMBChar = 0
+    end
+
+    def get_charset_name
+      return "utf-8"
+    end
+
+    def feed(aBuf)
+      aBuf.each_byte do |b|
+        c = b.chr
+        codingState = @_mCodingSM.next_state(c)
+        if codingState == EError
+          @_mState = ENotMe
+          break
+        elsif codingState == EItsMe
+          @_mState = EFoundIt
+          break
+        elsif codingState == EStart
+          if @_mCodingSM.get_current_charlen() >= 2
+            @_mNumOfMBChar += 1
+          end
+        end
+      end
+
+      if get_state() == EDetecting
+        if get_confidence() > SHORTCUT_THRESHOLD
+          @_mState = EFoundIt
+        end
+      end
+
+      return get_state()
+    end
+
+    def get_confidence
+      unlike = 0.99
+      if @_mNumOfMBChar < 6
+        for i in (0...@_mNumOfMBChar)
+          unlike = unlike * ONE_CHAR_PROB
+        end
+        return 1.0 - unlike
+      else
+        return unlike
+      end
+    end
+  end
+end