rchardet 1.3.1 → 1.4.0

data/lib/rchardet/codingstatemachine.rb

@@ -31,36 +31,37 @@ module CharDet
     attr_accessor :active
 
     def initialize(sm)
-      @_mModel = sm
-      @_mCurrentBytePos = 0
-      @_mCurrentCharLen = 0
+      @model = sm
+      @currentBytePos = 0
+      @currentCharLen = 0
       reset()
     end
 
     def reset
-      @_mCurrentState = EStart
+      @currentState = EStart
     end
 
     def next_state(c)
       # for each byte we get its class
       # if it is first byte, we also get byte length
-      byteCls = @_mModel['classTable'][c[0]]
-      if @_mCurrentState == EStart
-	@_mCurrentBytePos = 0
-	@_mCurrentCharLen = @_mModel['charLenTable'][byteCls]
+      b = c.bytes.first
+      byteCls = @model['classTable'][b]
+      if @currentState == EStart
+        @currentBytePos = 0
+        @currentCharLen = @model['charLenTable'][byteCls]
       end
       # from byte's class and stateTable, we get its next state
-      @_mCurrentState = @_mModel['stateTable'][@_mCurrentState * @_mModel['classFactor'] + byteCls]
-      @_mCurrentBytePos += 1
-      return @_mCurrentState
+      @currentState = @model['stateTable'][@currentState * @model['classFactor'] + byteCls]
+      @currentBytePos += 1
+      return @currentState
     end
 
     def get_current_charlen
-      return @_mCurrentCharLen
+      return @currentCharLen
     end
 
     def get_coding_state_machine
-      return @_mModel['name']
+      return @model['name']
     end
   end
 end

data/lib/rchardet/escprober.rb

@@ -30,58 +30,58 @@ module CharDet
   class EscCharSetProber < CharSetProber
     def initialize
       super()
-      @_mCodingSM = [ 
-	CodingStateMachine.new(HZSMModel),
-	CodingStateMachine.new(ISO2022CNSMModel),
-	CodingStateMachine.new(ISO2022JPSMModel),
-	CodingStateMachine.new(ISO2022KRSMModel)
-      ]
+      @codingSM = [ 
+                     CodingStateMachine.new(HZSMModel),
+                     CodingStateMachine.new(ISO2022CNSMModel),
+                     CodingStateMachine.new(ISO2022JPSMModel),
+                     CodingStateMachine.new(ISO2022KRSMModel)
+                    ]
       reset()
     end
 
     def reset
       super()
-      for codingSM in @_mCodingSM
-	next if not codingSM
-	codingSM.active = true
-	codingSM.reset()
+      for codingSM in @codingSM
+        next if !codingSM
+        codingSM.active = true
+        codingSM.reset()
       end
-      @_mActiveSM = @_mCodingSM.length
-      @_mDetectedCharset = nil
+      @activeSM = @codingSM.length
+      @detectedCharset = nil
     end
 
     def get_charset_name
-      return @_mDetectedCharset
+      return @detectedCharset
     end
 
     def get_confidence
-      if @_mDetectedCharset
-	return 0.99
+      if @detectedCharset
+        return 0.99
       else
-	return 0.00
+        return 0.00
       end
     end
 
     def feed(aBuf)
       aBuf.each_byte do |b|
-	c = b.chr
-	for codingSM in @_mCodingSM
-	  next unless codingSM
-	  next unless codingSM.active
-	  codingState = codingSM.next_state(c)
-	  if codingState == EError
-	    codingSM.active = false
-	    @_mActiveSM -= 1
-	    if @_mActiveSM <= 0
-	      @_mState = ENotMe
-	      return get_state()
-	    end
-	  elsif codingState == EItsMe
-	    @_mState = EFoundIt
-	    @_mDetectedCharset = codingSM.get_coding_state_machine()
-	    return get_state()
-	  end
-	end
+        c = b.chr
+        for codingSM in @codingSM
+          next unless codingSM
+          next unless codingSM.active
+          codingState = codingSM.next_state(c)
+          if codingState == EError
+            codingSM.active = false
+            @activeSM -= 1
+            if @activeSM <= 0
+              @state = ENotMe
+              return get_state()
+            end
+          elsif codingState == EItsMe
+            @state = EFoundIt
+            @detectedCharset = codingSM.get_coding_state_machine()
+            return get_state()
+          end
+        end
       end
 
       return get_state()

data/lib/rchardet/escsm.rb

@@ -7,6 +7,7 @@
 # the Initial Developer. All Rights Reserved.
 #
 # Contributor(s):
+#   Jeff Hodges - port to Ruby
 #   Mark Pilgrim - port to Python
 #
 # This library is free software; you can redistribute it and/or
@@ -59,7 +60,7 @@ module CharDet
     1,1,1,1,1,1,1,1,  # e8 - ef 
     1,1,1,1,1,1,1,1,  # f0 - f7 
     1,1,1,1,1,1,1,1,  # f8 - ff 
-  ]
+  ].freeze
 
   HZ_st = [
     EStart,EError,     3,EStart,EStart,EStart,EError,EError,# 00-07 
@@ -68,16 +69,16 @@ module CharDet
     5,EError,     6,EError,     5,     5,     4,EError,# 18-1f 
     4,EError,     4,     4,     4,EError,     4,EError,# 20-27 
     4,EItsMe,EStart,EStart,EStart,EStart,EStart,EStart,# 28-2f 
-  ]
+  ].freeze
 
-  HZCharLenTable = [0, 0, 0, 0, 0, 0]
+  HZCharLenTable = [0, 0, 0, 0, 0, 0].freeze
 
   HZSMModel = {'classTable' => HZ_cls,
-	     'classFactor' => 6,
-	     'stateTable' => HZ_st,
-	     'charLenTable' => HZCharLenTable,
-	     'name' => "HZ-GB-2312"
-}
+    'classFactor' => 6,
+    'stateTable' => HZ_st,
+    'charLenTable' => HZCharLenTable,
+    'name' => "HZ-GB-2312"
+  }.freeze
 
 ISO2022CN_cls = [
 2,0,0,0,0,0,0,0,  # 00 - 07 
@@ -112,7 +113,7 @@ ISO2022CN_cls = [
 2,2,2,2,2,2,2,2,  # e8 - ef 
 2,2,2,2,2,2,2,2,  # f0 - f7 
 2,2,2,2,2,2,2,2,  # f8 - ff 
-]
+].freeze
 
 ISO2022CN_st = [
 EStart,     3,EError,EStart,EStart,EStart,EStart,EStart,# 00-07 
@@ -123,16 +124,16 @@ EError,EError,EError,EItsMe,EError,EError,EError,EError,# 20-27
      5,     6,EError,EError,EError,EError,EError,EError,# 28-2f 
 EError,EError,EError,EItsMe,EError,EError,EError,EError,# 30-37 
 EError,EError,EError,EError,EError,EItsMe,EError,EStart,# 38-3f 
-]
+].freeze
 
-ISO2022CNCharLenTable = [0, 0, 0, 0, 0, 0, 0, 0, 0]
+ISO2022CNCharLenTable = [0, 0, 0, 0, 0, 0, 0, 0, 0].freeze
 
 ISO2022CNSMModel = {'classTable' => ISO2022CN_cls,
-		    'classFactor' => 9,
-		    'stateTable' => ISO2022CN_st,
-		    'charLenTable' => ISO2022CNCharLenTable,
-		    'name' => "ISO-2022-CN"
-}
+    'classFactor' => 9,
+    'stateTable' => ISO2022CN_st,
+    'charLenTable' => ISO2022CNCharLenTable,
+    'name' => "ISO-2022-CN"
+  }.freeze
 
 ISO2022JP_cls = [
 2,0,0,0,0,0,0,0,  # 00 - 07 
@@ -167,7 +168,7 @@ ISO2022JP_cls = [
 2,2,2,2,2,2,2,2,  # e8 - ef 
 2,2,2,2,2,2,2,2,  # f0 - f7 
 2,2,2,2,2,2,2,2,  # f8 - ff 
-]
+].freeze
 
 ISO2022JP_st = [ 
 EStart,     3,EError,EStart,EStart,EStart,EStart,EStart,# 00-07 
@@ -179,16 +180,16 @@ EError,EError,EError,     6,EItsMe,EError,EItsMe,EError,# 28-2f
 EError,EError,EError,EError,EError,EError,EItsMe,EItsMe,# 30-37 
 EError,EError,EError,EItsMe,EError,EError,EError,EError,# 38-3f 
 EError,EError,EError,EError,EItsMe,EError,EStart,EStart,# 40-47 
-]
+].freeze
 
-ISO2022JPCharLenTable = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+ISO2022JPCharLenTable = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0].freeze
 
 ISO2022JPSMModel = {'classTable' => ISO2022JP_cls,
-		    'classFactor' => 10,
-		    'stateTable' => ISO2022JP_st,
-		    'charLenTable' => ISO2022JPCharLenTable,
-		    'name' => "ISO-2022-JP"
-}
+    'classFactor' => 10,
+    'stateTable' => ISO2022JP_st,
+    'charLenTable' => ISO2022JPCharLenTable,
+    'name' => "ISO-2022-JP"
+  }.freeze
 
 ISO2022KR_cls = [
 2,0,0,0,0,0,0,0,  # 00 - 07 
@@ -223,7 +224,7 @@ ISO2022KR_cls = [
 2,2,2,2,2,2,2,2,  # e8 - ef 
 2,2,2,2,2,2,2,2,  # f0 - f7 
 2,2,2,2,2,2,2,2,  # f8 - ff 
-]
+].freeze
 
 ISO2022KR_st = [ 
 EStart,     3,EError,EStart,EStart,EStart,EError,EError,# 00-07 
@@ -231,14 +232,14 @@ EError,EError,EError,EError,EItsMe,EItsMe,EItsMe,EItsMe,# 08-0f
 EItsMe,EItsMe,EError,EError,EError,     4,EError,EError,# 10-17 
 EError,EError,EError,EError,     5,EError,EError,EError,# 18-1f 
 EError,EError,EError,EItsMe,EStart,EStart,EStart,EStart,# 20-27 
-]
+].freeze
 
-ISO2022KRCharLenTable = [0, 0, 0, 0, 0, 0]
+ISO2022KRCharLenTable = [0, 0, 0, 0, 0, 0].freeze
 
 ISO2022KRSMModel = {'classTable' => ISO2022KR_cls,
-		    'classFactor' => 6,
-		    'stateTable' => ISO2022KR_st,
-		    'charLenTable' => ISO2022KRCharLenTable,
-		    'name' => "ISO-2022-KR"
-}
+    'classFactor' => 6,
+    'stateTable' => ISO2022KR_st,
+    'charLenTable' => ISO2022KRCharLenTable,
+    'name' => "ISO-2022-KR"
+  }.freeze
 end

data/lib/rchardet/eucjpprober.rb

@@ -30,15 +30,15 @@ module CharDet
   class EUCJPProber < MultiByteCharSetProber
     def initialize
       super()
-      @_mCodingSM = CodingStateMachine.new(EUCJPSMModel)
-      @_mDistributionAnalyzer = EUCJPDistributionAnalysis.new()
-      @_mContextAnalyzer = EUCJPContextAnalysis.new()
+      @codingSM = CodingStateMachine.new(EUCJPSMModel)
+      @distributionAnalyzer = EUCJPDistributionAnalysis.new()
+      @contextAnalyzer = EUCJPContextAnalysis.new()
       reset
     end
 
     def reset
       super()
-      @_mContextAnalyzer.reset()
+      @contextAnalyzer.reset()
     end
 
     def get_charset_name
@@ -48,40 +48,40 @@ module CharDet
     def feed(aBuf)
       aLen = aBuf.length
       for i in (0...aLen)
-	codingState = @_mCodingSM.next_state(aBuf[i..i])
-	if codingState == EError
-	  $stderr << "#{get_charset_name} prober hit error at byte #{i}\n" if $debug
-	  @_mState = ENotMe
-	  break
-	elsif codingState == EItsMe
-	  @_mState = EFoundIt
-	  break
-	elsif codingState == EStart
-	  charLen = @_mCodingSM.get_current_charlen()
-	  if i == 0
-	    @_mLastChar[1] = aBuf[0..0]
-	    @_mContextAnalyzer.feed(@_mLastChar, charLen)
-	    @_mDistributionAnalyzer.feed(@_mLastChar, charLen)
-	  else
-	    @_mContextAnalyzer.feed(aBuf[i-1...i+1], charLen)
-	    @_mDistributionAnalyzer.feed(aBuf[i-1...i+1], charLen)
-	  end
-	end
+        codingState = @codingSM.next_state(aBuf[i, 1])
+        if codingState == EError
+          $stderr << "#{get_charset_name} prober hit error at byte #{i}\n" if $debug
+          @state = ENotMe
+          break
+        elsif codingState == EItsMe
+          @state = EFoundIt
+          break
+        elsif codingState == EStart
+          charLen = @codingSM.get_current_charlen()
+          if i == 0
+            @lastChar[1] = aBuf[0, 1]
+            @contextAnalyzer.feed(@lastChar, charLen)
+            @distributionAnalyzer.feed(@lastChar, charLen)
+          else
+            @contextAnalyzer.feed(aBuf[i-1, 2], charLen)
+            @distributionAnalyzer.feed(aBuf[i-1, 2], charLen)
+          end
+        end
       end
 
-      @_mLastChar[0] = aBuf[aLen-1..aLen-1]
+      @lastChar[0] = aBuf[aLen-1, 1]
 
       if get_state() == EDetecting
-	if @_mContextAnalyzer.got_enough_data() and (get_confidence() > SHORTCUT_THRESHOLD)
-	  @_mState = EFoundIt
-	end
+        if @contextAnalyzer.got_enough_data() and (get_confidence() > SHORTCUT_THRESHOLD)
+          @state = EFoundIt
+        end
       end
 
       return get_state()
     end
 
     def get_confidence
-      l = [@_mContextAnalyzer.get_confidence,@_mDistributionAnalyzer.get_confidence]
+      l = [@contextAnalyzer.get_confidence,@distributionAnalyzer.get_confidence]
       return l.max
     end
   end

data/lib/rchardet/euckrfreq.rb

@@ -7,6 +7,7 @@
 # the Initial Developer. All Rights Reserved.
 #
 # Contributor(s):
+#   Jeff Hodges - port to Ruby
 #   Mark Pilgrim - port to Python
 #
 # This library is free software; you can redistribute it and/or
@@ -592,5 +593,5 @@ EUCKRCharToFreqOrder = [
 8704,8705,8706,8707,8708,8709,8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,
 8720,8721,8722,8723,8724,8725,8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,
 8736,8737,8738,8739,8740,8741
-]
+].freeze
 end

data/lib/rchardet/euckrprober.rb

@@ -30,8 +30,8 @@ module CharDet
   class EUCKRProber < MultiByteCharSetProber
     def initialize
       super()
-      @_mCodingSM = CodingStateMachine.new(EUCKRSMModel)
-      @_mDistributionAnalyzer = EUCKRDistributionAnalysis.new()
+      @codingSM = CodingStateMachine.new(EUCKRSMModel)
+      @distributionAnalyzer = EUCKRDistributionAnalysis.new()
       reset()
     end
 

data/lib/rchardet/euctwfreq.rb

@@ -426,5 +426,6 @@ EUCTWCharToFreqOrder = [
 8678,8679,8680,8681,8682,8683,8684,8685,8686,8687,8688,8689,8690,8691,8692,8693, # 8694
 8694,8695,8696,8697,8698,8699,8700,8701,8702,8703,8704,8705,8706,8707,8708,8709, # 8710
 8710,8711,8712,8713,8714,8715,8716,8717,8718,8719,8720,8721,8722,8723,8724,8725, # 8726
-8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741] # 8742
+8726,8727,8728,8729,8730,8731,8732,8733,8734,8735,8736,8737,8738,8739,8740,8741  # 8742
+].freeze
 end

data/lib/rchardet/euctwprober.rb

@@ -30,8 +30,8 @@ module CharDet
   class EUCTWProber < MultiByteCharSetProber
     def initialize
       super()
-      @_mCodingSM = CodingStateMachine.new(EUCTWSMModel)
-      @_mDistributionAnalyzer = EUCTWDistributionAnalysis.new()
+      @codingSM = CodingStateMachine.new(EUCTWSMModel)
+      @distributionAnalyzer = EUCTWDistributionAnalysis.new()
       reset()
     end
 

data/lib/rchardet/gb2312freq.rb

@@ -469,6 +469,6 @@ GB2312CharToFreqOrder = [
 6271,3875,5768,6094,5034,5506,4376,5769,6761,2120,6476,5253,5770,6762,5771,5970,
 3990,5971,5557,5558,5772,6477,6095,2787,4641,5972,5121,6096,6097,6272,6763,3703,
 5867,5507,6273,4206,6274,4789,6098,6764,3619,3646,3833,3804,2394,3788,4936,3978,
-4866,4899,6099,6100,5559,6478,6765,3599,5868,6101,5869,5870,6275,6766,4527,6767]
-
+4866,4899,6099,6100,5559,6478,6765,3599,5868,6101,5869,5870,6275,6766,4527,6767
+].freeze
 end

data/lib/rchardet/gb2312prober.rb

@@ -30,8 +30,8 @@ module CharDet
   class GB2312Prober < MultiByteCharSetProber
     def initialize
       super
-      @_mCodingSM = CodingStateMachine.new(GB2312SMModel)
-      @_mDistributionAnalyzer = GB2312DistributionAnalysis.new()
+      @codingSM = CodingStateMachine.new(GB2312SMModel)
+      @distributionAnalyzer = GB2312DistributionAnalysis.new()
       reset()
     end
 

data/lib/rchardet/hebrewprober.rb

@@ -150,25 +150,25 @@ module CharDet
   class HebrewProber < CharSetProber
     def initialize
       super()
-      @_mLogicalProber = nil
-      @_mVisualProber = nil
+      @logicalProber = nil
+      @visualProber = nil
       reset()
     end
 
     def reset
-      @_mFinalCharLogicalScore = 0
-      @_mFinalCharVisualScore = 0
+      @finalCharLogicalScore = 0
+      @finalCharVisualScore = 0
       # The two last characters seen in the previous buffer,
       # mPrev and mBeforePrev are initialized to space in order to simulate a word 
       # delimiter at the beginning of the data
-      @_mPrev = ' '
-      @_mBeforePrev = ' '
+      @prev = ' '
+      @beforePrev = ' '
       # These probers are owned by the group prober.
     end
 
     def set_model_probers(logicalProber, visualProber)
-      @_mLogicalProber = logicalProber
-      @_mVisualProber = visualProber
+      @logicalProber = logicalProber
+      @visualProber = visualProber
     end
 
     def is_final(c)
@@ -215,34 +215,34 @@ module CharDet
       # so the word boundary detection works properly. [MAP]
 
       if get_state() == ENotMe
-	# Both model probers say it's not them. No reason to continue.
-	return ENotMe
+        # Both model probers say it's not them. No reason to continue.
+        return ENotMe
       end
 
       aBuf = filter_high_bit_only(aBuf)
 
       for cur in aBuf.split(' ')
-	if cur == ' '
-	  # We stand on a space - a word just ended
-	  if @_mBeforePrev != ' '
-	    # next-to-last char was not a space so self._mPrev is not a 1 letter word
-	    if is_final(@_mPrev)
-	      # case (1) [-2:not space][-1:final letter][cur:space]
-	      @_mFinalCharLogicalScore += 1
-	    elsif is_non_final(@_mPrev)
-	      # case (2) [-2:not space][-1:Non-Final letter][cur:space]
-	      @_mFinalCharVisualScore += 1
-	    end
-	  end
-	else
-	  # Not standing on a space
-	  if (@_mBeforePrev == ' ') and (is_final(@_mPrev)) and (cur != ' ')
-	    # case (3) [-2:space][-1:final letter][cur:not space]
-	    @_mFinalCharVisualScore += 1
-	  end
-	end
-	@_mBeforePrev = @_mPrev
-	@_mPrev = cur
+        if cur == ' '
+          # We stand on a space - a word just ended
+          if @beforePrev != ' '
+            # next-to-last char was not a space so self._mPrev is not a 1 letter word
+            if is_final(@prev)
+              # case (1) [-2:not space][-1:final letter][cur:space]
+              @finalCharLogicalScore += 1
+            elsif is_non_final(@prev)
+              # case (2) [-2:not space][-1:Non-Final letter][cur:space]
+              @finalCharVisualScore += 1
+            end
+          end
+        else
+          # Not standing on a space
+          if (@beforePrev == ' ') and (is_final(@prev)) and (cur != ' ')
+            # case (3) [-2:space][-1:final letter][cur:not space]
+            @finalCharVisualScore += 1
+          end
+        end
+        @beforePrev = @prev
+        @prev = cur
       end
 
       # Forever detecting, till the end or until both model probers return eNotMe (handled above)
@@ -252,26 +252,26 @@ module CharDet
     def get_charset_name
       # Make the decision: is it Logical or Visual?
       # If the final letter score distance is dominant enough, rely on it.
-      finalsub = @_mFinalCharLogicalScore - @_mFinalCharVisualScore
+      finalsub = @finalCharLogicalScore - @finalCharVisualScore
       if finalsub >= MIN_FINAL_CHAR_DISTANCE
-	return LOGICAL_HEBREW_NAME
+        return LOGICAL_HEBREW_NAME
       end
       if finalsub <= -MIN_FINAL_CHAR_DISTANCE
-	return VISUAL_HEBREW_NAME
+        return VISUAL_HEBREW_NAME
       end
 
       # It's not dominant enough, try to rely on the model scores instead.
-      modelsub = @_mLogicalProber.get_confidence() - @_mVisualProber.get_confidence()
+      modelsub = @logicalProber.get_confidence() - @visualProber.get_confidence()
       if modelsub > MIN_MODEL_DISTANCE
-	return LOGICAL_HEBREW_NAME
+        return LOGICAL_HEBREW_NAME
       end
       if modelsub < -MIN_MODEL_DISTANCE
-	return VISUAL_HEBREW_NAME
+        return VISUAL_HEBREW_NAME
       end
 
       # Still no good, back to final letter distance, maybe it'll save the day.
       if finalsub < 0.0
-	return VISUAL_HEBREW_NAME
+        return VISUAL_HEBREW_NAME
       end
 
       # (finalsub > 0 - Logical) or (don't know what to do) default to Logical.
@@ -280,8 +280,8 @@ module CharDet
 
     def get_state
       # Remain active as long as any of the model probers are active.
-      if (@_mLogicalProber.get_state() == ENotMe) and (@_mVisualProber.get_state() == ENotMe)
-	return ENotMe
+      if (@logicalProber.get_state() == ENotMe) and (@visualProber.get_state() == ENotMe)
+        return ENotMe
       end
       return EDetecting
     end