mullet 0.0.0 → 0.0.1

data/lib/mullet/html/parser/input_stream.rb

@@ -0,0 +1,711 @@
+require 'mullet/html/parser/constants'
+
+module Mullet; module HTML; module Parser
+
+  # Provides a unicode stream of characters to the HTMLTokenizer.
+
+  # This class takes care of character encoding and removing or replacing
+  # incorrect byte-sequences and also provides column and line tracking.
+
+  class InputStream
+
+    attr_accessor :queue, :char_encoding, :errors
+
+    # Initialises the InputStream.
+    # 
+    # InputStream(source, [encoding]) -> Normalized stream from source
+    # for use by the HTML5Lib.
+    # 
+    # source can be either a file-object, local filename or a string.
+    # 
+    # The optional encoding parameter must be a string that indicates
+    # the encoding.  If specified, that encoding will be used,
+    # regardless of any BOM or later declaration (such as in a meta
+    # element)
+    #  
+    # parseMeta - Look for a <meta> element containing encoding information
+
+    def initialize(source, options = {})
+      @encoding   = nil
+      @parse_meta = true
+      @chardet    = true
+
+      options.each {|name, value| instance_variable_set("@#{name}", value) }
+
+      # partial Ruby 1.9 support
+      if @encoding and source.respond_to? :force_encoding
+        source.force_encoding(@encoding) rescue nil
+      end
+
+      # Raw Stream
+      @raw_stream = open_stream(source)
+
+      # Encoding Information
+      #Number of bytes to use when looking for a meta element with
+      #encoding information
+      @NUM_BYTES_META = 512
+      #Number of bytes to use when using detecting encoding using chardet
+      @NUM_BYTES_CHARDET = 256
+      #Number of bytes to use when reading content
+      @NUM_BYTES_BUFFER = 1024
+
+      #Encoding to use if no other information can be found
+      @DEFAULT_ENCODING = 'windows-1252'
+
+      #Detect encoding iff no explicit "transport level" encoding is supplied
+      if @encoding.nil?
+        @char_encoding = detect_encoding
+      else
+        @char_encoding = @encoding
+      end
+
+      # Read bytes from stream decoding them into Unicode
+      @buffer = @raw_stream.read(@NUM_BYTES_BUFFER) || ''
+      if @char_encoding == 'windows-1252'
+        @win1252 = true
+      elsif @char_encoding != 'utf-8'
+        require 'iconv'
+        begin
+          @buffer << @raw_stream.read unless @raw_stream.eof?
+          @buffer = Iconv.iconv('utf-8', @char_encoding, @buffer).first
+        rescue
+          @win1252 = true
+        end
+      end
+
+      @queue = []
+      @errors = []
+
+      # Reset position in the list to read from
+      @tell = 0
+      @line = @col = 0
+      @line_lengths = []
+    end
+
+    # Produces a file object from source.
+    #
+    # source can be either a file object, local filename or a string.
+    def open_stream(source)
+      # Already an IO like object
+      if source.respond_to?(:read)
+        source
+      else
+        # Treat source as a string and wrap in StringIO
+        StringIO.new(source)
+      end
+    end
+
+    def detect_encoding
+
+      #First look for a BOM
+      #This will also read past the BOM if present
+      encoding = detect_bom
+
+      #If there is no BOM need to look for meta elements with encoding 
+      #information
+      if encoding.nil? and @parse_meta
+        encoding = detect_encoding_meta
+      end
+
+      #Guess with chardet, if avaliable
+      if encoding.nil? and @chardet
+        begin
+          require 'rubygems'
+          require 'UniversalDetector' # gem install chardet
+          buffers = []
+          detector = UniversalDetector::Detector.instance
+          detector.reset
+          until @raw_stream.eof?
+            buffer = @raw_stream.read(@NUM_BYTES_CHARDET)
+            break if !buffer or buffer.empty?
+            buffers << buffer
+            detector.feed(buffer)
+            break if detector.instance_eval {@done}
+            detector.instance_eval {
+              @_mLastChar = @_mLastChar.chr if Fixnum === @_mLastChar
+            }
+          end
+          detector.close
+          encoding = detector.result['encoding']
+          seek(buffers*'', 0)
+        rescue LoadError
+        end
+      end
+
+      # If all else fails use the default encoding
+      if encoding.nil?
+        encoding = @DEFAULT_ENCODING
+      end
+    
+      #Substitute for equivalent encoding
+      if 'iso-8859-1' == encoding.downcase
+        encoding = 'windows-1252'
+      end
+
+      encoding
+    end
+
+    # Attempts to detect at BOM at the start of the stream. If
+    # an encoding can be determined from the BOM return the name of the
+    # encoding otherwise return nil
+    def detect_bom
+      bom_dict = {
+        "\xef\xbb\xbf"     => 'utf-8',
+        "\xff\xfe"         => 'utf-16le',
+        "\xfe\xff"         => 'utf-16be',
+        "\xff\xfe\x00\x00" => 'utf-32le',
+        "\x00\x00\xfe\xff" => 'utf-32be'
+      }
+
+      # Go to beginning of file and read in 4 bytes
+      string = @raw_stream.read(4)
+      return nil unless string
+
+      # Try detecting the BOM using bytes from the string
+      encoding = bom_dict[string[0...3]]      # UTF-8
+      seek = 3
+      unless encoding
+        # Need to detect UTF-32 before UTF-16
+        encoding = bom_dict[string]       # UTF-32
+        seek = 4
+        unless encoding
+          encoding = bom_dict[string[0...2]]  # UTF-16
+          seek = 2
+        end
+      end
+
+      # Set the read position past the BOM if one was found, otherwise
+      # set it to the start of the stream
+      seek(string, encoding ? seek : 0)
+
+      return encoding
+    end
+
+    def seek(buffer, n)
+      if @raw_stream.respond_to?(:unget)
+        @raw_stream.unget(buffer[n..-1])
+        return
+      end
+
+      if @raw_stream.respond_to?(:seek)
+        begin
+          @raw_stream.seek(n)
+          return
+        rescue Errno::ESPIPE
+        end
+      end
+
+      #TODO: huh?
+      require 'delegate'
+      @raw_stream = SimpleDelegator.new(@raw_stream)
+
+      class << @raw_stream
+        def read(chars=-1)
+          if chars == -1 or chars > @data.length
+            result = @data
+            @data = ''
+            return result if __getobj__.eof?
+            return result + __getobj__.read if chars == -1
+            return result + __getobj__.read(chars-result.length)
+          elsif @data.empty?
+            return __getobj__.read(chars)
+          else
+            result = @data[1...chars]
+            @data = @data[chars..-1]
+            return result
+          end
+        end
+
+        def unget(data)
+          if !@data or @data.empty?
+            @data = data
+          else
+            @data += data
+          end
+        end
+      end
+
+      @raw_stream.unget(buffer[n .. -1])
+    end
+
+    # Report the encoding declared by the meta element
+    def detect_encoding_meta
+      buffer = @raw_stream.read(@NUM_BYTES_META)
+      parser = EncodingParser.new(buffer)
+      seek(buffer, 0)
+      return parser.get_encoding
+    end
+
+    # Returns (line, col) of the current position in the stream.
+    def position
+      line, col = @line, @col
+      if @queue and @queue.last != :EOF
+        @queue.reverse.each do |c|
+          if c == "\n"
+            line -= 1
+            raise RuntimeError.new("col=#{col}") unless col == 0
+            col = @line_lengths[line]
+          else
+            col -= 1
+          end 
+        end
+      end
+      return [line + 1, col]
+    end
+
+    # Read one character from the stream or queue if available. Return
+    # EOF when EOF is reached.
+    def char
+      unless @queue.empty?
+        return @queue.shift
+      else
+        if @tell + 3 > @buffer.length && !@raw_stream.eof?
+          # read next block
+          @buffer = @buffer[@tell..-1] + @raw_stream.read(@NUM_BYTES_BUFFER)
+          @tell = 0
+        end
+
+        c = @buffer[@tell]
+        @tell += 1
+
+        case c
+
+        when String
+          # partial Ruby 1.9 support
+          case c
+          when "\0"
+            @errors.push("null-character")
+            c = "\uFFFD" # null characters are invalid
+          when "\r"
+            @tell += 1 if @buffer[@tell] == "\n"
+            c = "\n"
+          when "\x80" .. "\x9F"
+            c = ENTITIES_WINDOWS1252[c.ord-0x80].chr('utf-8')
+          when "\xA0" .. "\xFF"
+            if c.encoding == Encoding::ASCII_8BIT
+              c = c.encode('utf-8','iso-8859-1')
+            end
+          end
+
+          if c == "\x0D"
+            # normalize newlines
+            @tell += 1 if @buffer[@tell] == 0x0A
+            c = 0x0A
+          end
+
+          # update position in stream
+          if c == "\x0a"
+            @line_lengths << @col
+            @line += 1
+            @col = 0
+          else
+            @col += 1
+          end
+
+          c
+
+        when 0x01..0x7F
+          if c == 0x0D
+            # normalize newlines
+            @tell += 1 if @buffer[@tell] == 0x0A
+            c = 0x0A
+          end
+
+          # update position in stream
+          if c == 0x0a
+            @line_lengths << @col
+            @line += 1
+            @col = 0
+          else
+            @col += 1
+          end
+
+          c.chr
+
+        when 0x80..0xBF
+          if !@win1252
+            [0xFFFD].pack('U') # invalid utf-8
+          elsif c <= 0x9f
+            [ENTITIES_WINDOWS1252[c-0x80]].pack('U')
+          else
+            "\xC2" + c.chr # convert to utf-8
+          end
+
+        when 0xC0..0xFF
+          if instance_variable_defined?("@win1252") && @win1252
+            "\xC3" + (c - 64).chr # convert to utf-8
+          # from http://www.w3.org/International/questions/qa-forms-utf-8.en.php
+          elsif @buffer[@tell - 1..@tell + 3] =~ /^
+                ( [\xC2-\xDF][\x80-\xBF]             # non-overlong 2-byte
+                |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs
+                | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}  # straight 3-byte
+                |  \xED[\x80-\x9F][\x80-\xBF]        # excluding surrogates
+                |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
+                | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
+                |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
+                )/x
+            @tell += $1.length - 1
+            $1
+          else
+            [0xFFFD].pack('U') # invalid utf-8
+          end
+
+        when 0x00
+          @errors.push("null-character")
+          [0xFFFD].pack('U') # null characters are invalid
+
+        else
+          :EOF
+        end
+      end
+    end
+
+    # Returns a string of characters from the stream up to but not
+    # including any character in characters or EOF. characters can be
+    # any container that supports the in method being called on it.
+    def chars_until(characters, opposite=false)
+      char_stack = [char]
+
+      while char_stack.last != :EOF
+        break unless (characters.include?(char_stack.last)) == opposite
+        char_stack.push(char)
+      end
+
+      # Put the character stopped on back to the front of the queue
+      # from where it came.
+      c = char_stack.pop
+      @queue.insert(0, c) unless c == :EOF
+      return char_stack.join('')
+    end
+
+    def unget(characters)
+      return if characters == :EOF
+      if characters.respond_to? :to_a
+        @queue.unshift(*characters.to_a)
+      else
+        characters.reverse.each_char {|c| @queue.unshift(c)}
+      end
+    end
+  end
+
+  # String-like object with an assosiated position and various extra methods
+  # If the position is ever greater than the string length then an exception is raised
+  class EncodingBytes < String
+
+    attr_accessor :position
+
+    def initialize(value)
+      super(value)
+      @position = -1
+    end
+  
+    def each
+      while @position < length
+        @position += 1
+        yield self[@position]
+      end
+    rescue EOF
+    end
+  
+    def current_byte
+      raise EOF if @position >= length
+      return self[@position].chr
+    end
+  
+    # Skip past a list of characters
+    def skip(chars=SPACE_CHARACTERS)
+      while chars.include?(current_byte)
+        @position += 1
+      end
+    end
+
+    # Look for a sequence of bytes at the start of a string. If the bytes 
+    # are found return true and advance the position to the byte after the 
+    # match. Otherwise return false and leave the position alone
+    def match_bytes(bytes, lower=false)
+      data = self[position ... position+bytes.length]
+      data.downcase! if lower
+      rv = (data == bytes)
+      @position += bytes.length if rv == true
+      return rv
+    end
+  
+    # Look for the next sequence of bytes matching a given sequence. If
+    # a match is found advance the position to the last byte of the match
+    def jump_to(bytes)
+      new_position = self[position .. -1].index(bytes)
+      if new_position
+        @position += (new_position + bytes.length-1)
+        return true
+      else
+        raise EOF
+      end
+    end
+  
+    # Move the pointer so it points to the next byte in a set of possible
+    # bytes
+    def find_next(byte_list)
+      until byte_list.include?(current_byte)
+        @position += 1
+      end
+    end
+  end
+
+  # Mini parser for detecting character encoding from meta elements
+  class EncodingParser
+    ASCII_PUNCTUATION = %r{[\x09-\x0D\x20-\x2F\x3A-\x40\x5B-\x60\x7B-\x7E]}
+    # a (hopefully) temporary hack to deal with the fact that ruby doesn't have a built in encodings
+    #   library
+    ENCODINGS = ['euc_jp', 'utf-8', "iso8859-2", "iso-8859-1", "utf-16", "UTF-16LE", "UTF-16BE"].inject({}){|m, v| m[v.downcase.gsub(ASCII_PUNCTUATION, '')] = v; m}
+    # string - the data to work on for encoding detection
+    def initialize(data)
+      @data = EncodingBytes.new(data.to_s)
+      @encoding = nil
+    end
+
+    @@method_dispatch = [
+      ['<!--', :handle_comment],
+      ['<meta', :handle_meta],
+      ['</', :handle_possible_end_tag],
+      ['<!', :handle_other],
+      ['<?', :handle_other],
+      ['<', :handle_possible_start_tag]
+    ]
+
+    def get_encoding
+      @data.each do |byte|
+        keep_parsing = true
+        @@method_dispatch.each do |(key, method)|
+          if @data.match_bytes(key, lower = true)
+            keep_parsing = send(method)
+            break
+          end
+        end
+        break unless keep_parsing
+      end
+
+      unless @encoding.nil?
+        @encoding = @encoding.strip
+        if ["utf16", "utf16be", "utf16le", "utf32", "utf32be", "utf32le"].include?(@encoding.downcase.gsub(ASCII_PUNCTUATION, ''))
+          @encoding = 'utf-8'
+        end
+      end
+      
+      return @encoding
+    end
+
+    # Skip over comments
+    def handle_comment
+      return @data.jump_to('-->')
+    end
+
+    def handle_meta
+      # if we have <meta not followed by a space so just keep going
+      return true unless SPACE_CHARACTERS.include?(@data.current_byte)
+
+      #We have a valid meta element we want to search for attributes
+      while true
+        #Try to find the next attribute after the current position
+        attr = get_attribute
+
+        return true if attr.nil?
+        if attr[0] == 'charset'
+          tentative_encoding = attr[1]
+          codec = codec_name(tentative_encoding)
+          if codec
+            @encoding = codec
+            return false
+          end
+        elsif attr[0] == 'content'
+          content_parser = ContentAttrParser.new(EncodingBytes.new(attr[1]))
+          tentative_encoding = content_parser.parse
+          codec = codec_name(tentative_encoding)
+          if codec
+            @encoding = codec
+            return false
+          end
+        end
+      end
+    end
+
+    def handle_possible_start_tag
+      return handle_possible_tag(false)
+    end
+
+    def handle_possible_end_tag
+      @data.position += 1
+      return handle_possible_tag(true)
+    end
+
+    def handle_possible_tag(end_tag)
+      unless ASCII_LETTERS.include?(@data.current_byte)
+        #If the next byte is not an ascii letter either ignore this
+        #fragment (possible start tag case) or treat it according to 
+        #handleOther
+        if end_tag
+          @data.position -= 1
+          handle_other
+        end
+        return true
+      end
+    
+      @data.find_next(SPACE_CHARACTERS + ['<', '>'])
+
+      if @data.current_byte == '<'
+        #return to the first step in the overall "two step" algorithm
+        #reprocessing the < byte
+        @data.position -= 1  
+      else
+        #Read all attributes
+        {} until get_attribute.nil?
+      end
+      return true
+    end
+
+    def handle_other
+      return @data.jump_to('>')
+    end
+
+    # Return a name,value pair for the next attribute in the stream,
+    # if one is found, or nil
+    def get_attribute
+      @data.skip(SPACE_CHARACTERS + ['/'])
+
+      if @data.current_byte == '<'
+        @data.position -= 1
+        return nil
+      elsif @data.current_byte == '>'
+        return nil
+      end
+
+      attr_name = []
+      attr_value = []
+      space_found = false
+      #Step 5 attribute name
+      while true
+        if @data.current_byte == '=' and attr_name
+          break
+        elsif SPACE_CHARACTERS.include?(@data.current_byte)
+          space_found = true
+          break
+        elsif ['/', '<', '>'].include?(@data.current_byte)
+          return [attr_name.join(''), '']
+        elsif ASCII_UPPERCASE.include?(@data.current_byte)
+          attr_name.push(@data.current_byte.downcase)
+        else
+          attr_name.push(@data.current_byte)
+        end
+        #Step 6
+        @data.position += 1
+      end
+      #Step 7
+      if space_found
+        @data.skip
+        #Step 8
+        unless @data.current_byte == '='
+          @data.position -= 1
+          return [attr_name.join(''), '']
+        end
+      end
+      #XXX need to advance position in both spaces and value case
+      #Step 9
+      @data.position += 1
+      #Step 10
+      @data.skip
+      #Step 11
+      if ["'", '"'].include?(@data.current_byte)
+        #11.1
+        quote_char = @data.current_byte
+        while true
+          @data.position+=1
+          #11.3
+          if @data.current_byte == quote_char
+            @data.position += 1
+            return [attr_name.join(''), attr_value.join('')]
+          #11.4
+          elsif ASCII_UPPERCASE.include?(@data.current_byte)
+            attr_value.push(@data.current_byte.downcase)
+          #11.5
+          else
+            attr_value.push(@data.current_byte)
+          end
+        end
+      elsif ['>', '<'].include?(@data.current_byte)
+        return [attr_name.join(''), '']
+      elsif ASCII_UPPERCASE.include?(@data.current_byte)
+        attr_value.push(@data.current_byte.downcase)
+      else
+        attr_value.push(@data.current_byte)
+      end
+      while true
+        @data.position += 1
+        if (SPACE_CHARACTERS + ['>', '<']).include?(@data.current_byte)
+          return [attr_name.join(''), attr_value.join('')]
+        elsif ASCII_UPPERCASE.include?(@data.current_byte)
+          attr_value.push(@data.current_byte.downcase)
+        else
+          attr_value.push(@data.current_byte)
+        end
+      end
+    end
+
+    def codec_name(encoding)
+      if (!encoding.nil? && encoding.kind_of?(String))
+        canonical_name = encoding.downcase.gsub(ASCII_PUNCTUATION, '')
+        ENCODINGS[canonical_name]
+        # p encoding
+        # encoding
+      else
+        nil
+      end
+    end
+  end
+
+  class ContentAttrParser
+    def initialize(data)
+      @data = data
+    end
+
+    def parse
+      begin
+        #Skip to the first ";"
+        @data.position = 0
+        @data.jump_to(';')
+        @data.position += 1
+        @data.skip
+        #Check if the attr name is charset 
+        #otherwise return
+        @data.jump_to('charset')
+        @data.position += 1
+        @data.skip
+        unless @data.current_byte == '='
+          #If there is no = sign keep looking for attrs
+          return nil
+        end
+        @data.position += 1
+        @data.skip
+        #Look for an encoding between matching quote marks
+        if ['"', "'"].include?(@data.current_byte)
+          quote_mark = @data.current_byte
+          @data.position += 1
+          old_position = @data.position
+          @data.jump_to(quote_mark)
+          return @data[old_position ... @data.position]
+        else
+          #Unquoted value
+          old_position = @data.position
+          begin
+            @data.find_next(SPACE_CHARACTERS)
+            return @data[old_position ... @data.position]
+          rescue EOF
+            #Return the whole remaining value
+            return @data[old_position .. -1]
+          end
+        end
+      rescue EOF
+        return nil
+      end
+    end
+  end
+
+end; end; end