chardet 0.9.0

data/lib/UniversalDetector.rb

@@ -0,0 +1,209 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Hui (zhengzhengzheng@gmail.com) - port to Ruby
+#   Mark Pilgrim - first port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+# 
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+# 
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+require "EscCharSetProber"
+require "MBCSGroupProber"
+require "SBCSGroupProber"
+require "Latin1Prober"
+require "singleton"
+
+module UniversalDetector    
+  
+    class << self
+        def encoding(data)
+            chardet(data)['encoding']
+        end
+        
+        def chardet(data)
+            u = UniversalDetector::Detector.instance
+            u.reset()
+            u.feed(data)
+            u.close()
+            u.result
+        end
+    end
+    
+    DEBUG = nil
+  
+    Detectiong = 0
+    FoundIt = 1
+    NotMe = 2
+
+    Start = 0
+    Error = 1
+    ItsMe = 2
+  
+    MINIMUM_THRESHOLD = 0.20
+    PureAscii = 0
+    EscAscii = 1
+    Highbyte = 2
+
+    SHORTCUT_THRESHOLD = 0.95
+  
+    class Detector
+        
+        include Singleton
+        
+        attr_reader :result
+    
+        def initialize
+            @_highBitDetector = /[\x80-\xFF]/
+            @_escDetector = /\033|~\{/
+            @_mEscCharSetProber = nil
+            @_mCharSetProbers = []
+            reset
+        end
+    
+        def reset
+            @result = {"encoding"=> nil, "confidence"=> 0.0}
+            @done = false
+            @_mStart = true
+            @_mGotData = false
+            @_mInputState = :PureAscii
+            @_mLastChar = ""
+            if @_mEscCharSetProber
+                @_mEscCharSetProber.reset
+            end
+            for prober in @_mCharSetProbers
+                prober.reset
+            end
+        end    
+    
+        def feed(data)
+            if @done || data.empty?
+                return 
+            end
+            unless  @_mGotData
+                # If the data starts with BOM, we know it is UTF
+                if data[0,3] == "\xEF\xBB\xBF":
+                    # EF BB BF  UTF-8 with BOM
+                    @result = {"encoding"=> "UTF-8", "confidence"=> 1.0}
+                elsif data[0,4] == "\xFF\xFE\x00\x00":
+                    # FF FE 00 00  UTF-32, little-endian BOM
+                    @result = {"encoding"=> "UTF-32LE", "confidence"=> 1.0}
+                elsif data[0,4] == "\x00\x00\xFE\xFF": 
+                    # 00 00 FE FF  UTF-32, big-endian BOM
+                    @result = {"encoding"=> "UTF-32BE", "confidence"=> 1.0}
+                elsif data[0,4] == "\xFE\xFF\x00\x00":
+                    # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
+                    @result = {"encoding"=> "X-ISO-10646-UCS-4-3412", "confidence"=> 1.0}
+                elsif data[0,4] == "\x00\x00\xFF\xFE":
+                    # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
+                    @result = {"encoding"=> "X-ISO-10646-UCS-4-2143", "confidence"=> 1.0}
+                elsif data[0,4] == "\xFF\xFE":
+                    # FF FE  UTF-16, little endian BOM
+                    @result = {"encoding"=> "UTF-16LE", "confidence"=> 1.0}
+                elsif data[0,2] == "\xFE\xFF":
+                    # FE FF  UTF-16, big endian BOM
+                    @result = {"encoding"=> "UTF-16BE", "confidence"=> 1.0}          
+                end
+            end
+            @_mGotData = true
+            if @result["encoding"] && @result["confidence"] > 0.0
+                @done = true
+                return
+            end            
+            
+            if @_mInputState == :PureAscii
+                if data =~ @_highBitDetector
+                    @_mInputState = :Highbyte
+                elsif (@_mLastChar + data) =~ @_escDetector
+                    @_mInputState = :EscAscii
+                end
+            end                        
+            
+            @_mLastChar = data[-1]
+            if @_mInputState == :EscAscii
+                unless @_mEscCharSetProber
+                    @_mEscCharSetProber = EscCharSetProber.new
+                end
+                if @_mEscCharSetProber.feed(data) == constants.eFoundIt
+                    @result = {"encoding"=> @_mEscCharSetProber.get_charset_name() ,"confidence"=> @_mEscCharSetProber.get_confidence()}
+                    @done = true          
+                end  
+            elsif @_mInputState == :Highbyte
+                if @_mCharSetProbers.empty?
+                    @_mCharSetProbers = MBCSGroupProber.new.mProbers + SBCSGroupProber.new.mProbers + [Latin1Prober.new]
+                end                                
+                @_mCharSetProbers.each do |prober|                    
+                    if prober.feed(data) == :FoundIt
+                        @result = {"encoding"=> prober.get_charset_name(), "confidence"=> prober.get_confidence()}
+                        @done = true
+                        break
+                    end
+                end #for
+            end
+        end #feed
+  
+        def close
+            if @done then return end
+            unless @_mGotData
+                if DEBUG
+                    p("no data received!\n")
+                end
+                return
+            end
+            @done = true
+            
+            if @_mInputState == :PureAscii
+                @result = {"encoding" =>  "ascii", "confidence" => 1.0}
+                return @result
+            end
+
+            if @_mInputState == :Highbyte
+                proberConfidence = nil
+                maxProberConfidence = 0.0
+                maxProber = nil
+                for prober in @_mCharSetProbers
+                    unless prober then next end
+                    proberConfidence = prober.get_confidence()
+                    if proberConfidence > maxProberConfidence
+                        maxProberConfidence = proberConfidence
+                        maxProber = prober
+                    end
+                end
+                if maxProber and (maxProberConfidence > MINIMUM_THRESHOLD)
+                    @result = {"encoding" => maxProber.get_charset_name(),
+                                   "confidence" => maxProber.get_confidence()}
+                    return @result
+                end
+            end #if
+            
+            if DEBUG
+                p("no probers hit minimum threshhold\n")
+                for prober in @_mCharSetProbers:
+                    unless prober then next end
+                    p("%s confidence = %s\n" % \
+                                     [prober.get_charset_name(), \
+                                      prober.get_confidence()])
+                end
+            end            
+        end #close
+    end #class
+                       
+end #module

data/python-docs/css/chardet.css

@@ -0,0 +1,299 @@
+html {
+  margin: 0;
+  padding: 0;
+}
+
+body {
+  background-color: #fff;
+  color: #333;
+  font-family: 'Lucida Grande', Verdana, Geneva, Lucida, Helvetica, sans-serif;
+  font-size: 100%;
+  margin: 10px;
+  padding: 0;
+}
+
+a:link, a:visited { 
+  background-color: transparent;
+  color: #333;
+  text-decoration: none !important;
+  border-bottom: 1px dotted #333 !important;
+  text-decoration: underline;
+  border-bottom: 0;
+}
+
+a:hover {
+  background-color: transparent;
+  color: #993344;
+  text-decoration: none !important;
+  text-decoration: underline;
+  border-bottom: 1px dotted #993344 !important;
+  border-bottom: 0;
+}
+
+h1 {
+  margin: 8px 0 0 0;
+  padding: 0;
+  font-variant: small-caps;
+  letter-spacing: 0.1em;
+  font-family: "Book Antiqua", Georgia, Palatino, Times, "Times New Roman", serif;
+}
+
+h1 a:link, h1 a:visited, h1 a:hover {
+  background-color: transparent ! important;
+  color: #333 ! important;
+  text-decoration: none ! important;
+  border-bottom: 0px ! important;
+}
+
+#intro { 
+  width: 730px;
+ }
+
+#intro ul {
+  margin-left: 0;
+  padding-left: 0;
+  display: inline;
+}
+
+#intro ul li {
+  display: inline;
+  font-size: small;
+}
+
+#intro ul li.li1 {
+}
+
+#intro p {
+  font-size: small;
+  font-weight: normal;
+  margin: 1.2em 0 0 0;
+  padding: 0;
+}
+
+.z {
+  float:left;
+  background: url(/img/shadowAlpha.png) no-repeat bottom right !important;
+  background: url(/img/shadow.gif) no-repeat bottom right;
+  margin: 15px 0 0 10px !important;
+  margin: 15px 0 0 5px;
+}
+
+.z .sectionInner {
+  background: none !important;
+  background: url(/img/shadow2.gif) no-repeat left top;
+  padding: 0 !important;
+  padding: 0 6px 6px 0;
+  }
+
+.z .sectionInner .sectionInner2 {
+  background-color: #fff;
+  border: 1px solid #a9a9a9;
+  padding: 4px;
+  margin: -6px 6px 6px -6px !important;
+  margin: 0;
+}
+
+.s { 
+  margin-left: 1em;
+  margin-right: 1em;
+  margin-bottom: 1em;
+}
+
+#main {
+  clear: left;
+  margin-left: 11px;
+  margin-bottom: 2em;
+  font-size: small;
+}
+
+#mainInner { 
+  margin-left: 1em;
+  margin-bottom: 2em;
+  padding-top: 1em;
+}
+
+.footernavigation {
+  clear: both;
+  font-size: small;
+  padding-bottom: 1em;
+  margin-bottom: 0;
+}
+
+.example, .section, .appendix {
+  line-height: 150%;
+}
+
+#breadcrumb {
+  width: 100%;
+  margin: 0 0 1em 0;
+  padding: 0;
+  line-height: 140%;
+  font-size: small;
+}
+
+#breadcrumb #thispage {
+  font-weight: bold;
+}
+
+/* ----- Python code syntax coloring ----- */
+.computeroutput, .traceback, .pykeyword, .pystring, .pycomment, .pyfunction, .pyclass {
+  background-color: white;
+}
+
+.pykeyword, .pyfunction, .pyclass {
+  font-weight: bold;
+}
+
+.computeroutput {
+  color: teal;
+}
+
+.traceback {
+  color: red;
+}
+
+.pykeyword {
+  color: navy;
+}
+
+.pystring {
+  color: olive;
+}
+
+.pycomment {
+  color: green;
+  font-style: italic;
+}
+
+.pyfunction {
+  color: teal;
+}
+
+.pyclass {
+  color: blue;
+}
+
+/* ----- standard stuff ----- */
+.skip {
+  display: none;
+}
+
+samp, code, tt, pre  {
+  font-weight: normal;
+  font-family: monospace;
+  font-size: small;
+}
+
+img {
+  border: 0;
+}
+
+acronym, abbr {
+/*  border-bottom: 1px dotted #333;*/
+  border-bottom: 0;
+  font-style: normal;
+  cursor: help;
+}
+
+hr {
+  clear: both;
+  margin-top: 2em !important;
+  margin-top: 1em;
+  height: 1px;
+  background-color: #cecbc6;
+  color: #cecbc6;
+}
+
+#footer {
+  text-align: center;
+  font-size: x-small;
+}
+
+body.docs .example {
+  border-left: 4px double #ddd !important;
+  border-left: 1px solid #ccc;
+  margin-left: 2em;
+  padding-left: 2em;
+}
+
+body.docs .example h3 {
+  font-size: 100%;
+}
+
+body.docs .example a.skip:link,
+body.docs .example a.skip:visited,
+body.docs .example a.skip:hover,
+body.docs .section h3.title a.skip:link,
+body.docs .section h3.title a.skip:visited,
+body.docs .section h3.title a.skip:hover,
+body.docs .appendix h3.title a.skip:link,
+body.docs .appendix h3.title a.skip:visited,
+body.docs .appendix h3.title a.skip:hover {
+  display: block;
+  float: left;
+  vertical-align: bottom;
+  text-decoration: none;
+  border-bottom: 0 ! important;
+  margin-right: 6px;
+}
+
+.reference-from h3,
+.seealso h3,
+.furtherreading h3 {
+  margin-top: -1.2em;
+  margin-left: -15px;
+  font-size: small;
+  width: 8em;
+  border: 1px solid #a9a9a9;
+  padding: 3px 3px 3px 13px;
+  background: white;
+  position: relative;
+}
+
+.reference-from, .seealso, .furtherreading {
+  width: 680px;
+  margin-top: 3em;
+  margin-bottom: 3em;
+  border: 1px solid #a9a9a9;
+}
+
+table.tip, table.note, table.warning, table.caution, table.important {
+  margin-bottom: 1em;
+}
+
+.table h3 {
+  display: none;
+}
+
+.table table td {
+  padding: 5px 1em 5px 1em;
+}
+
+div.download {
+  width: 708px;
+  margin-top: 3em;
+  margin-bottom: 3em;
+  border: 1px solid #a9a9a9;
+}
+
+div.download h3 {
+  margin-top: -1.2em;
+  margin-left: -15px;
+  font-size: small;
+  width: 10em;
+  border: 1px solid #a9a9a9;
+  padding: 3px 3px 3px 13px;
+  background-color: #fff;
+  color: #222;
+  position: relative;
+}
+
+div.download p {
+  margin-left: 1em;
+}
+
+div.download ul {
+  list-style: none;
+  padding-left: 1em;
+  margin-left: 0;
+}