chardet2 1.0.0 → 1.0.1

data/lib/CharDistributionAnalysis.rb

@@ -58,7 +58,9 @@ module  UniversalDetector
             #"""feed a character with known length"""
             if aCharLen == 2
                 # we only care about 2-bytes character in our distribution analysis
-                order = get_order(aStr)
+                b1 = aStr.get_byte(0)
+                b2 = aStr.get_byte(1)
+                order = get_order([b1, b2])
             else
                 order = -1
             end
@@ -242,4 +244,4 @@ module  UniversalDetector
         end
     end
     
-end
+end

data/lib/EUCJPProber.rb

@@ -55,7 +55,7 @@ module  UniversalDetector
         def feed(aBuf)
             aLen = aBuf.length
             for i in 0...aLen
-                codingState = @_mCodingSM.next_state(aBuf[i])
+                codingState = @_mCodingSM.next_state(aBuf.get_byte(i))
                 if codingState == :Error
                     if DEBUG
                         p(get_charset_name() + ' prober hit error at byte ' + i.to_s + '\n')
@@ -94,4 +94,4 @@ module  UniversalDetector
             return [contxtCf, distribCf].max
         end
     end
-end
+end

data/lib/HebrewProber.rb

@@ -224,7 +224,7 @@ module  UniversalDetector
 
             aBuf = filter_high_bit_only(aBuf)
 
-            for cur in aBuf
+            aBuf.each_char do |cur|
                 if cur == ' '
                     # We stand on a space - a word just ended
                     if @_mBeforePrev != ' '
@@ -289,4 +289,4 @@ module  UniversalDetector
             return :Detecting
         end
     end
-end
+end

data/lib/JapaneseContextAnalysis.rb

@@ -145,7 +145,7 @@ module UniversalDetector
             # this character will simply our logic and improve performance.
             i = @_mNeedToSkipCharNum
             while i < aLen
-                order, charLen = get_order(aBuf[i..i+2])
+                order, charLen = get_order(aBuf[i..i+2].to_bytes)
                 i += charLen
                 if i > aLen
                     @_mNeedToSkipCharNum = i - aLen
@@ -195,10 +195,10 @@ module UniversalDetector
 
             # return its order if it is hiragana
             if aStr.length > 1
-                if (aStr[0] == '\202') and \
+                if (aStr[0] == 0x82) and \
                    (aStr[1] >= 0x9F) and \
                    (aStr[1] <= 0xF1)
-                    return ord(aStr[1]) - 0x9F, charLen
+                    return aStr[1] - 0x9F, charLen
                 end
             end
 
@@ -210,7 +210,6 @@ module UniversalDetector
         def get_order(aStr)
             unless aStr then return -1, 1 end
             # find out current char's byte length
-            aStr = aStr.to_s
             if (aStr[0] == 0x8E) or ((aStr[0] >= 0xA1) and (aStr[0] <= 0xFE))
                 charLen = 2
             elsif aStr[0] == 0x8F
@@ -224,7 +223,7 @@ module UniversalDetector
                 if (aStr[0] == 0xA4) and \
                    (aStr[1] >= 0xA1) and \
                    (aStr[1] <= 0xF3)
-                    return aStr[1][0] - 0xA1, charLen
+                    return aStr[1] - 0xA1, charLen
                 end
             end
 

data/lib/Latin1Prober.rb

@@ -118,8 +118,8 @@ module UniversalDetector
 
         def feed(aBuf)
             aBuf = filter_with_english_letters(aBuf)
-            for c in aBuf
-                charClass = Latin1_CharToClass[c[0]]
+            aBuf.each_char do |c|
+                charClass = Latin1_CharToClass[c.get_byte(0)]
                 freq = Latin1ClassModel[(@_mLastCharClass * CLASS_NUM) + charClass]
                 if freq == 0
                     @_mState = :NotMe

data/lib/MultiByteCharSetProber.rb

@@ -56,7 +56,7 @@ module  UniversalDetector
         def feed(aBuf)            
             aLen = aBuf.length
             for i in 0...aLen
-                codingState = @_mCodingSM.next_state(aBuf[i])
+                codingState = @_mCodingSM.next_state(aBuf.get_byte(i))
                 if codingState == :Error
                     if UniversalDetector::DEBUG
                         p(get_charset_name() + ' prober hit error at byte ' + i.to_s + '\n')
@@ -91,4 +91,4 @@ module  UniversalDetector
             return @_mDistributionAnalyzer.get_confidence()
         end
     end
-end
+end

data/lib/SJISProber.rb

@@ -55,7 +55,7 @@ module UniversalDetector
         def feed(aBuf)
             aLen = aBuf.length
             for i in 0...aLen
-                codingState = @_mCodingSM.next_state(aBuf[i])
+                codingState = @_mCodingSM.next_state(aBuf.get_byte(i))
                 if codingState == :Error
                     if DEBUG
                         p(get_charset_name() + ' prober hit error at byte ' + i.to_s + '\n')
@@ -96,4 +96,4 @@ module UniversalDetector
             return [contxtCf, distribCf].max
         end
     end
-end
+end

data/lib/SingleByteCharSetProber.rb

@@ -75,7 +75,7 @@ module UniversalDetector
             end                        
             
             for i in 0...aLen
-                c = aBuf[i]
+                c = aBuf.get_byte(i)
                 order = @_mModel['charToOrderMap'][c]
                 if order < SYMBOL_CAT_ORDER
                     @_mTotalChar += 1
@@ -128,4 +128,4 @@ module UniversalDetector
             return r
         end
     end
-end
+end

data/lib/UTF8Prober.rb

@@ -52,9 +52,9 @@ module UniversalDetector
         end
 
         def feed(aBuf) 
-            aLen = aBuf.length       
+            aLen = aBuf.length
             for i in 0...aLen
-                codingState = @_mCodingSM.next_state(aBuf[i])                
+                codingState = @_mCodingSM.next_state(aBuf.get_byte(i))
                 if codingState == :Error
                     @_mState = :NotMe
                     break

data/lib/UniversalDetector.rb

@@ -30,15 +30,16 @@ require "EscCharSetProber"
 require "MBCSGroupProber"
 require "SBCSGroupProber"
 require "Latin1Prober"
+require "shim"
 require "singleton"
 
-module UniversalDetector    
-  
+module UniversalDetector
+
     class << self
         def encoding(data)
             chardet(data)['encoding']
         end
-        
+
         def chardet(data)
             u = UniversalDetector::Detector.instance
             u.reset()

data/lib/shim.rb

@@ -0,0 +1,37 @@
+module UniversalDetector
+
+  def self.is18?
+    RUBY_VERSION =~ /^1\.8/
+  end
+
+end
+
+class String
+
+  if UniversalDetector.is18?
+    alias :get_byte :[]
+  else
+    def get_byte(i)
+      self[i].ord
+    end
+  end
+
+  def to_bytes
+    bytes.to_a
+  end
+
+end
+
+class Array
+
+  def get_byte(i)
+    v = self[i]
+    v = v.bytes.to_a.first if v.is_a?(String)
+    v
+  end
+
+  def to_bytes
+    map {|v| v.is_a?(String) ? v.get_byte(0) : v}
+  end
+
+end

metadata

@@ -1,31 +1,43 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: chardet2
-version: !ruby/object:Gem::Version
-  version: 1.0.0
+version: !ruby/object:Gem::Version 
+  hash: 21
   prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 1
+  version: 1.0.1
 platform: ruby
-authors:
+authors: 
 - Jan Xie
 - Felipe Tanus
 - Hui
 autorequire: UniversalDetector
 bindir: bin
 cert_chain: []
-date: 2013-05-17 00:00:00.000000000 Z
+
+date: 2013-05-17 00:00:00 +08:00
+default_executable: 
 dependencies: []
+
 description: 
-email:
+email: 
 - jan.h.xie@gmail.com
 executables: []
+
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - lib/MBCSSM.rb
 - lib/MultiByteCharSetProber.rb
 - lib/JapaneseContextAnalysis.rb
 - lib/LangCyrillicModel.rb
 - lib/EUCKRFreq.rb
 - lib/GB2312Freq.rb
+- lib/shim.rb
 - lib/EUCKRProber.rb
 - lib/CodingStateMachine.rb
 - lib/LangHungarianModel.rb
@@ -55,29 +67,39 @@ files:
 - lib/CharSetProber.rb
 - COPYING
 - README.markdown
+has_rdoc: true
 homepage: https://github.com/janx/chardet
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
-summary: Character encoding auto-detection in Ruby, compatible with 1.9/2.0. Base
-  on Mark Pilgrim's Python port and Hui's ruby port.
+summary: Character encoding auto-detection in Ruby, compatible with 1.9/2.0. Base on Mark Pilgrim's Python port and Hui's ruby port.
 test_files: []
+