gigo 1.4.0 → 2.0.0

data/lib/gigo/rchardet/sbcharsetprober.rb

@@ -1,125 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-module GIGO
-module CharDet
-  SAMPLE_SIZE = 64
-  SB_ENOUGH_REL_THRESHOLD = 1024
-  POSITIVE_SHORTCUT_THRESHOLD = 0.95
-  NEGATIVE_SHORTCUT_THRESHOLD = 0.05
-  SYMBOL_CAT_ORDER = 250
-  NUMBER_OF_SEQ_CAT = 4
-  POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1
-  #NEGATIVE_CAT = 0
-
-  class SingleByteCharSetProber < CharSetProber
-    def initialize(model, reversed=false, nameProber=nil)
-      super()
-      @_mModel = model
-      @_mReversed = reversed # TRUE if we need to reverse every pair in the model lookup
-      @_mNameProber = nameProber # Optional auxiliary prober for name decision
-      reset()
-    end
-
-    def reset
-      super()
-      @_mLastOrder = 255 # char order of last character
-      @_mSeqCounters = [0] * NUMBER_OF_SEQ_CAT
-      @_mTotalSeqs = 0
-      @_mTotalChar = 0
-      @_mFreqChar = 0 # characters that fall in our sampling range
-    end
-
-    def get_charset_name
-      if @_mNameProber
-	      return @_mNameProber.get_charset_name()
-      else
-	      return @_mModel['charsetName']
-      end
-    end
-
-    def feed(aBuf)
-      if not @_mModel['keepEnglishLetter']
-	      aBuf = filter_without_english_letters(aBuf)
-      end
-      aLen = aBuf.length
-      if not aLen
-	      return get_state()
-      end
-      for c in aBuf.split('')
-        char = c.respond_to?(:bytes) ? c.bytes.first : c[0]
-      	order = @_mModel['charToOrderMap'][char]
-      	if order < SYMBOL_CAT_ORDER
-      	  @_mTotalChar += 1
-      	end
-      	if order < SAMPLE_SIZE
-      	  @_mFreqChar += 1
-      	  if @_mLastOrder < SAMPLE_SIZE
-      	    @_mTotalSeqs += 1
-      	    if not @_mReversed
-      	      @_mSeqCounters[@_mModel['precedenceMatrix'][(@_mLastOrder * SAMPLE_SIZE) + order]] += 1
-      	    else # reverse the order of the letters in the lookup
-      	      @_mSeqCounters[@_mModel['precedenceMatrix'][(order * SAMPLE_SIZE) + @_mLastOrder]] += 1
-      	    end
-      	  end
-      	end
-      	@_mLastOrder = order
-      end
-
-      if get_state == EDetecting
-      	if @_mTotalSeqs > SB_ENOUGH_REL_THRESHOLD
-      	  cf = get_confidence
-      	  if cf > POSITIVE_SHORTCUT_THRESHOLD
-      	    $stderr << "#{@_mModel['charsetName']} confidence = #{cf}, we have a winner\n" if $debug
-      	    @_mState = EFoundIt
-      	  elsif cf < NEGATIVE_SHORTCUT_THRESHOLD
-      	    $stderr << "#{@_mModel['charsetName']} confidence = #{cf}, below negative shortcut threshold #{NEGATIVE_SHORTCUT_THRESHOLD}\n" if $debug
-      	    @_mState = ENotMe
-      	  end
-      	end
-      end
-
-      return get_state
-    end
-
-    def get_confidence
-      r = 0.01
-      if @_mTotalSeqs > 0
-      	#            print self._mSeqCounters[POSITIVE_CAT], self._mTotalSeqs, self._mModel['mTypicalPositiveRatio']
-      	r = (1.0 * @_mSeqCounters[POSITIVE_CAT]) / @_mTotalSeqs / @_mModel['mTypicalPositiveRatio']
-      	#            print r, self._mFreqChar, self._mTotalChar
-      	r = r * @_mFreqChar / @_mTotalChar
-      	if r >= 1.0
-      	  r = 0.99
-      	end
-      end
-      return r
-    end
-  end
-end
-end

data/lib/gigo/rchardet/sbcsgroupprober.rb

@@ -1,59 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-module GIGO
-module CharDet
-  class SBCSGroupProber < CharSetGroupProber
-    def initialize
-      super
-      @_mProbers = [ 
-	SingleByteCharSetProber.new(Win1251CyrillicModel),
-	SingleByteCharSetProber.new(Koi8rModel),
-	SingleByteCharSetProber.new(Latin5CyrillicModel),
-	SingleByteCharSetProber.new(MacCyrillicModel),
-	SingleByteCharSetProber.new(Ibm866Model),
-	SingleByteCharSetProber.new(Ibm855Model),
-	SingleByteCharSetProber.new(Latin7GreekModel),
-	SingleByteCharSetProber.new(Win1253GreekModel),
-	SingleByteCharSetProber.new(Latin5BulgarianModel),
-	SingleByteCharSetProber.new(Win1251BulgarianModel),
-	SingleByteCharSetProber.new(Latin2HungarianModel),
-	SingleByteCharSetProber.new(Win1250HungarianModel),
-	SingleByteCharSetProber.new(TIS620ThaiModel),
-      ]
-      hebrewProber = HebrewProber.new()
-      logicalHebrewProber = SingleByteCharSetProber.new(Win1255HebrewModel, false, hebrewProber)
-      visualHebrewProber = SingleByteCharSetProber.new(Win1255HebrewModel, true, hebrewProber)
-      hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber)
-      @_mProbers += [hebrewProber, logicalHebrewProber, visualHebrewProber]
-
-      reset()
-    end
-  end
-end
-end

data/lib/gigo/rchardet/sjisprober.rb

@@ -1,89 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-module GIGO
-module CharDet
-  class SJISProber < MultiByteCharSetProber
-    def initialize
-      super()
-      @_mCodingSM = CodingStateMachine.new(SJISSMModel)
-      @_mDistributionAnalyzer = SJISDistributionAnalysis.new()
-      @_mContextAnalyzer = SJISContextAnalysis.new()
-      reset()
-    end
-
-    def reset
-      super()
-      @_mContextAnalyzer.reset()
-    end
-
-    def get_charset_name
-      return "SHIFT_JIS"
-    end
-
-    def feed(aBuf)
-      aLen = aBuf.length
-      for i in (0...aLen)
-      	codingState = @_mCodingSM.next_state(aBuf[i..i])
-      	if codingState == EError
-      	  $stderr << "#{get_charset_name} prober hit error at byte #{i}\n" if $debug
-      	  @_mState = ENotMe
-      	  break
-      	elsif codingState == EItsMe
-      	  @_mState = EFoundIt
-      	  break
-      	elsif codingState == EStart
-      	  charLen = @_mCodingSM.get_current_charlen()
-      	  if i == 0
-      	    @_mLastChar[1] = aBuf[0..0]
-      	    @_mContextAnalyzer.feed(@_mLastChar[2 - charLen..-1], charLen)
-      	    @_mDistributionAnalyzer.feed(@_mLastChar, charLen)
-      	  else
-      	    @_mContextAnalyzer.feed(aBuf[i + 1 - charLen ... i + 3 - charLen], charLen)
-      	    @_mDistributionAnalyzer.feed(aBuf[i - 1 ... i + 1], charLen)
-      	  end
-      	end
-      end
-
-      @_mLastChar[0] = aBuf[aLen - 1.. aLen-1]
-
-      if get_state() == EDetecting
-      	if @_mContextAnalyzer.got_enough_data() and (get_confidence() > SHORTCUT_THRESHOLD)
-      	  @_mState = EFoundIt
-      	end
-      end
-
-      return get_state()
-    end
-
-    def get_confidence
-      l = [@_mContextAnalyzer.get_confidence(), @_mDistributionAnalyzer.get_confidence()]
-      return l.max
-    end
-  end
-end
-end

data/lib/gigo/rchardet/universaldetector.rb

@@ -1,169 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-module GIGO
-module CharDet
-  MINIMUM_THRESHOLD = 0.20
-  EPureAscii = 0
-  EEscAscii = 1
-  EHighbyte = 2
-
-  class UniversalDetector
-    attr_accessor :result
-    def initialize
-      @_highBitDetector = Regexp.new '[\x80-\xFF]', nil, 'n'
-      @_escDetector = Regexp.new '(\033|\~\{)', nil, 'n'
-      @_mEscCharSetProber = nil
-      @_mCharSetProbers = []
-      reset()
-    end
-
-    def reset
-      @result = {'encoding' => nil, 'confidence' => 0.0}
-      @done = false
-      @_mStart = true
-      @_mGotData = false
-      @_mInputState = EPureAscii
-      @_mLastChar = ''
-      if @_mEscCharSetProber
-	      @_mEscCharSetProber.reset
-      end
-      for prober in @_mCharSetProbers
-	      prober.reset
-      end
-    end
-
-    def feed(aBuf)
-      # HACK: force internal encoding to be ascii-8bit (ruby 1.8 bytes)
-      aBuf = aBuf.force_encoding('ASCII-8BIT') if aBuf.respond_to?(:force_encoding)
-      return if @done
-
-      aLen = aBuf.length
-      return if not aLen
-
-      if not @_mGotData
-      	# If the data starts with BOM, we know it is UTF
-      	if aBuf[0...3] == "\xEF\xBB\xBF"
-      	  # EF BB BF  UTF-8 with BOM
-      	  @result = {'encoding' => "UTF-8", 'confidence' => 1.0}
-      	elsif aBuf[0...4] == "\xFF\xFE\x00\x00"
-      	  # FF FE 00 00  UTF-32, little-endian BOM
-      	  @result = {'encoding' => "UTF-32LE", 'confidence' => 1.0}
-      	elsif aBuf[0...4] == "\x00\x00\xFE\xFF"
-      	  # 00 00 FE FF  UTF-32, big-endian BOM
-      	  @result = {'encoding' => "UTF-32BE", 'confidence' => 1.0}
-      	elsif aBuf[0...4] == "\xFE\xFF\x00\x00"
-      	  # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
-      	  @result = {'encoding' => "X-ISO-10646-UCS-4-3412", 'confidence' => 1.0}
-      	elsif aBuf[0...4] == "\x00\x00\xFF\xFE"
-      	  # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
-      	  @result = {'encoding' =>  "X-ISO-10646-UCS-4-2143", 'confidence' =>  1.0}
-      	elsif aBuf[0...2] == "\xFF\xFE"
-      	  # FF FE  UTF-16, little endian BOM
-      	  @result = {'encoding' =>  "UTF-16LE", 'confidence' =>  1.0}
-      	elsif aBuf[0...2] == "\xFE\xFF"
-      	  # FE FF  UTF-16, big endian BOM
-      	  @result = {'encoding' =>  "UTF-16BE", 'confidence' =>  1.0}
-      	end
-      end
-
-      @_mGotData = true
-      if @result['encoding'] and (@result['confidence'] > 0.0)
-      	@done = true
-      	return
-      end
-      if @_mInputState == EPureAscii
-      	if @_highBitDetector =~ (aBuf)
-      	  @_mInputState = EHighbyte
-      	elsif (@_mInputState == EPureAscii) and @_escDetector =~ (@_mLastChar + aBuf)
-      	  @_mInputState = EEscAscii
-      	end
-      end
-      
-      @_mLastChar = aBuf[-1..-1]
-      if @_mInputState == EEscAscii
-      	if not @_mEscCharSetProber
-      	  @_mEscCharSetProber = EscCharSetProber.new
-      	end
-      	if @_mEscCharSetProber.feed(aBuf) == EFoundIt
-      	  @result = {'encoding' =>  self._mEscCharSetProber.get_charset_name(),
-      			       'confidence' =>  @_mEscCharSetProber.get_confidence()
-      	  }
-      	  @done = true
-      	end
-      elsif @_mInputState == EHighbyte
-      	if not @_mCharSetProbers or @_mCharSetProbers.empty?
-      	  @_mCharSetProbers = [MBCSGroupProber.new(), SBCSGroupProber.new(), Latin1Prober.new()]
-      	end
-      	for prober in @_mCharSetProbers
-      	  if prober.feed(aBuf) == EFoundIt
-      	    @result = {'encoding' =>  prober.get_charset_name(),
-      				   'confidence' =>  prober.get_confidence()}
-      	    @done = true
-      	    break
-      	  end
-      	end
-      end
-
-    end
-
-    def close
-      return if @done
-      if not @_mGotData
-      	$stderr << "no data received!\n" if $debug
-      	return
-      end
-      @done = true
-
-      if @_mInputState == EPureAscii
-      	@result = {'encoding' => 'ascii', 'confidence' => 1.0}
-      	return @result
-      end
-
-      if @_mInputState == EHighbyte
-      	confidences = {}
-              @_mCharSetProbers.each{ |prober| confidences[prober] = prober.get_confidence }
-      	maxProber = @_mCharSetProbers.max{ |a,b| confidences[a] <=> confidences[b] }
-      	if maxProber and maxProber.get_confidence > MINIMUM_THRESHOLD
-      	  @result = {'encoding' =>  maxProber.get_charset_name(),
-      			       'confidence' =>  maxProber.get_confidence()}
-      	  return @result
-      	end
-      end
-
-      if $debug
-      	$stderr << "no probers hit minimum threshhold\n" if $debug
-      	for prober in @_mCharSetProbers[0]._mProbers
-      	  next if not prober
-      	  $stderr << "#{prober.get_charset_name} confidence = #{prober.get_confidence}\n" if $debug
-      	end
-      end
-    end
-  end
-end
-end

data/lib/gigo/rchardet/utf8prober.rb

@@ -1,87 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Jeff Hodges - port to Ruby
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-# 
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-# 
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-module GIGO
-module CharDet
-  ONE_CHAR_PROB = 0.5
-
-  class UTF8Prober < CharSetProber
-    def initialize
-      super()
-      @_mCodingSM = CodingStateMachine.new(UTF8SMModel)
-      reset()
-    end
-
-    def reset
-      super()
-      @_mCodingSM.reset()
-      @_mNumOfMBChar = 0
-    end
-
-    def get_charset_name
-      return "utf-8"
-    end
-
-    def feed(aBuf)
-      for c in aBuf.split('')
-      	codingState = @_mCodingSM.next_state(c)
-      	if codingState == EError
-      	  @_mState = ENotMe
-      	  break
-      	elsif codingState == EItsMe
-      	  @_mState = EFoundIt
-      	  break
-      	elsif codingState == EStart
-      	  if @_mCodingSM.get_current_charlen() >= 2
-      	    @_mNumOfMBChar += 1
-      	  end
-      	end
-      end
-
-      if get_state == EDetecting
-      	if get_confidence > SHORTCUT_THRESHOLD
-      	  @_mState = EFoundIt
-      	end
-      end
-
-      return get_state
-    end
-
-    def get_confidence
-      unlike = 0.99
-      if @_mNumOfMBChar < 6
-      	for i in (0...@_mNumOfMBChar)
-      	  unlike = unlike * ONE_CHAR_PROB
-      	end
-      	return 1.0 - unlike
-            else
-      	return unlike
-      end
-    end
-  end
-end
-end