nokogiri-maglev- 1.5.0.1

data/lib/nokogiri/css/tokenizer.rex

@@ -0,0 +1,55 @@
+module Nokogiri
+module CSS
+class Tokenizer
+
+macro
+  nl        \n|\r\n|\r|\f
+  w         [\s]*
+  nonascii  [^\0-\177]
+  num       -?([0-9]+|[0-9]*\.[0-9]+)
+  unicode   \\[0-9A-Fa-f]{1,6}(\r\n|[\s])?
+
+  escape    {unicode}|\\[^\n\r\f0-9A-Fa-f]
+  nmchar    [_A-Za-z0-9-]|{nonascii}|{escape}
+  nmstart   [_A-Za-z]|{nonascii}|{escape}
+  ident     [-@]?({nmstart})({nmchar})*
+  name      ({nmchar})+
+  string1   "([^\n\r\f"]|{nl}|{nonascii}|{escape})*"
+  string2   '([^\n\r\f']|{nl}|{nonascii}|{escape})*'
+  string    {string1}|{string2}
+
+rule
+
+# [:state]  pattern  [actions]
+
+            has\({w}         { [:HAS, text] }
+            {ident}\({w}     { [:FUNCTION, text] }
+            {ident}          { [:IDENT, text] }
+            \#{name}         { [:HASH, text] }
+            {w}~={w}         { [:INCLUDES, text] }
+            {w}\|={w}        { [:DASHMATCH, text] }
+            {w}\^={w}        { [:PREFIXMATCH, text] }
+            {w}\$={w}        { [:SUFFIXMATCH, text] }
+            {w}\*={w}        { [:SUBSTRINGMATCH, text] }
+            {w}!={w}         { [:NOT_EQUAL, text] }
+            {w}={w}          { [:EQUAL, text] }
+            {w}\)            { [:RPAREN, text] }
+            {w}\[{w}         { [:LSQUARE, text] }
+            {w}\]            { [:RSQUARE, text] }
+            {w}\+{w}         { [:PLUS, text] }
+            {w}>{w}          { [:GREATER, text] }
+            {w},{w}          { [:COMMA, text] }
+            {w}~{w}          { [:TILDE, text] }
+            \:not\({w}       { [:NOT, text] }
+            {num}            { [:NUMBER, text] }
+            {w}\/\/{w}       { [:DOUBLESLASH, text] }
+            {w}\/{w}         { [:SLASH, text] }
+            
+            U\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?  {[:UNICODE_RANGE, text] }
+            
+            [\s]+            { [:S, text] }
+            {string}         { [:STRING, text] }
+            .                { [text, text] }
+end
+end
+end

data/lib/nokogiri/css/xpath_visitor.rb

@@ -0,0 +1,171 @@
+module Nokogiri
+  module CSS
+    class XPathVisitor # :nodoc:
+      def visit_function node
+        #  note that nth-child and nth-last-child are preprocessed in css/node.rb.
+        msg = :"visit_function_#{node.value.first.gsub(/[(]/, '')}"
+        return self.send(msg, node) if self.respond_to?(msg)
+
+        case node.value.first
+        when /^text\(/
+          'child::text()'
+        when /^self\(/
+          "self::#{node.value[1]}"
+        when /^eq\(/
+          "position() = #{node.value[1]}"
+        when /^(nth|nth-of-type|nth-child)\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
+            an_plus_b(node.value[1])
+          else
+            "position() = #{node.value[1]}"
+          end
+        when /^(nth-last-child|nth-last-of-type)\(/
+          if node.value[1].is_a?(Nokogiri::CSS::Node) and node.value[1].type == :AN_PLUS_B
+            an_plus_b(node.value[1], :last => true)
+          else
+            index = node.value[1].to_i - 1
+            index == 0 ? "position() = last()" : "position() = last() - #{index}"
+          end
+        when /^(first|first-of-type)\(/
+          "position() = 1"
+        when /^(last|last-of-type)\(/
+          "position() = last()"
+        when /^contains\(/
+          "contains(., #{node.value[1]})"
+        when /^gt\(/
+          "position() > #{node.value[1]}"
+        when /^only-child\(/
+          "last() = 1"
+        when /^comment\(/
+          "comment()"
+        when /^has\(/
+          node.value[1].accept(self)
+        else
+          args = ['.'] + node.value[1..-1]
+          "#{node.value.first}#{args.join(', ')})"
+        end
+      end
+
+      def visit_not node
+        child = node.value.first
+        if :ELEMENT_NAME == child.type
+          "not(self::#{child.accept(self)})"
+        else
+          "not(#{child.accept(self)})"
+        end
+      end
+
+      def visit_id node
+        node.value.first =~ /^#(.*)$/
+        "@id = '#{$1}'"
+      end
+
+      def visit_attribute_condition node
+         attribute = if (node.value.first.type == :FUNCTION) or (node.value.first.value.first =~ /::/)
+                       ''
+                     else
+                       '@'
+                     end
+        attribute += node.value.first.accept(self)
+
+        # Support non-standard css
+        attribute.gsub!(/^@@/, '@')
+
+        return attribute unless node.value.length == 3
+
+        value = node.value.last
+        value = "'#{value}'" if value !~ /^['"]/
+
+        case node.value[1]
+        when :equal
+          attribute + " = " + "#{value}"
+        when :not_equal
+          attribute + " != " + "#{value}"
+        when :substring_match
+          "contains(#{attribute}, #{value})"
+        when :prefix_match
+          "starts-with(#{attribute}, #{value})"
+        when :dash_match
+          "#{attribute} = #{value} or starts-with(#{attribute}, concat(#{value}, '-'))"
+        when :includes
+          "contains(concat(\" \", #{attribute}, \" \"),concat(\" \", #{value}, \" \"))"
+        when :suffix_match
+          "substring(#{attribute}, string-length(#{attribute}) - " +
+            "string-length(#{value}) + 1, string-length(#{value})) = #{value}"
+        else
+          attribute + " #{node.value[1]} " + "#{value}"
+        end
+      end
+
+      def visit_pseudo_class node
+        if node.value.first.is_a?(Nokogiri::CSS::Node) and node.value.first.type == :FUNCTION
+          node.value.first.accept(self)
+        else
+          msg = :"visit_pseudo_class_#{node.value.first.gsub(/[(]/, '')}"
+          return self.send(msg, node) if self.respond_to?(msg)
+
+          case node.value.first
+          when "first", "first-child" then "position() = 1"
+          when "last", "last-child" then "position() = last()"
+          when "first-of-type" then "position() = 1"
+          when "last-of-type" then "position() = last()"
+          when "only-of-type" then "last() = 1"
+          when "empty" then "not(node())"
+          when "parent" then "node()"
+          when "root" then "not(parent::*)"
+          else
+            node.value.first + "(.)"
+          end
+        end
+      end
+
+      def visit_class_condition node
+        "contains(concat(' ', @class, ' '), ' #{node.value.first} ')"
+      end
+
+      {
+        'combinator'                => ' and ',
+        'direct_adjacent_selector'  => "/following-sibling::*[1]/self::",
+        'preceding_selector'        => "/following-sibling::",
+        'descendant_selector'       => '//',
+        'child_selector'            => '/',
+      }.each do |k,v|
+        class_eval %{
+          def visit_#{k} node
+            "\#{node.value.first.accept(self)}#{v}\#{node.value.last.accept(self)}"
+          end
+        }
+      end
+
+      def visit_conditional_selector node
+        node.value.first.accept(self) + '[' +
+        node.value.last.accept(self) + ']'
+      end
+
+      def visit_element_name node
+        node.value.first
+      end
+
+      def accept node
+        node.accept(self)
+      end
+
+    private
+      def an_plus_b node, options={}
+        raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
+
+        a = node.value[0].to_i
+        b = node.value[3].to_i
+        position = options[:last] ? "(last()-position()+1)" : "position()"
+
+        if (b == 0)
+          return "(#{position} mod #{a}) = 0"
+        else
+          compare = (a < 0) ? "<=" : ">="
+          return "(#{position} #{compare} #{b}) and (((#{position}-#{b}) mod #{a.abs}) = 0)"
+        end
+      end
+
+    end
+  end
+end

data/lib/nokogiri/decorators/slop.rb

@@ -0,0 +1,35 @@
+module Nokogiri
+  module Decorators
+    ###
+    # The Slop decorator implements method missing such that a methods may be
+    # used instead of XPath or CSS.  See Nokogiri.Slop
+    module Slop
+      ###
+      # look for node with +name+.  See Nokogiri.Slop
+      def method_missing name, *args, &block
+        prefix = implied_xpath_context
+
+        if args.empty?
+          list = xpath("#{prefix}#{name.to_s.sub(/^_/, '')}")
+        elsif args.first.is_a? Hash
+          hash = args.first
+          if hash[:css]
+            list = css("#{name}#{hash[:css]}")
+          elsif hash[:xpath]
+            conds = Array(hash[:xpath]).join(' and ')
+            list = xpath("#{prefix}#{name}[#{conds}]")
+          end
+        else
+          CSS::Parser.without_cache do
+            list = xpath(
+              *CSS.xpath_for("#{name}#{args.first}", :prefix => prefix)
+            )
+          end
+        end
+
+        super if list.empty?
+        list.length == 1 ? list.first : list
+      end
+    end
+  end
+end

data/lib/nokogiri/html.rb

@@ -0,0 +1,36 @@
+require 'nokogiri/html/entity_lookup'
+require 'nokogiri/html/document'
+require 'nokogiri/html/document_fragment'
+require 'nokogiri/html/sax/parser_context'
+require 'nokogiri/html/sax/parser'
+require 'nokogiri/html/element_description'
+require 'nokogiri/html/element_description_defaults'
+
+module Nokogiri
+  class << self
+    ###
+    # Parse HTML.  Convenience method for Nokogiri::HTML::Document.parse
+    def HTML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
+      Nokogiri::HTML::Document.parse(thing, url, encoding, options, &block)
+    end
+  end
+
+  module HTML
+    class << self
+      ###
+      # Parse HTML.  Convenience method for Nokogiri::HTML::Document.parse
+      def parse thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block
+        Document.parse(thing, url, encoding, options, &block)
+      end
+
+      ####
+      # Parse a fragment from +string+ in to a NodeSet.
+      def fragment string, encoding = nil
+        HTML::DocumentFragment.parse string, encoding
+      end
+    end
+
+    # Instance of Nokogiri::HTML::EntityLookup
+    NamedCharacters = EntityLookup.new
+  end
+end

data/lib/nokogiri/html/builder.rb

@@ -0,0 +1,35 @@
+module Nokogiri
+  module HTML
+    ###
+    # Nokogiri HTML builder is used for building HTML documents.  It is very
+    # similar to the Nokogiri::XML::Builder.  In fact, you should go read the
+    # documentation for Nokogiri::XML::Builder before reading this
+    # documentation.
+    #
+    # == Synopsis:
+    #
+    # Create an HTML document with a body that has an onload attribute, and a
+    # span tag with a class of "bold" that has content of "Hello world".
+    #
+    #   builder = Nokogiri::HTML::Builder.new do |doc|
+    #     doc.html {
+    #       doc.body(:onload => 'some_func();') {
+    #         doc.span.bold {
+    #           doc.text "Hello world"
+    #         }
+    #       }
+    #     }
+    #   end
+    #   puts builder.to_html
+    #
+    # The HTML builder inherits from the XML builder, so make sure to read the
+    # Nokogiri::XML::Builder documentation.
+    class Builder < Nokogiri::XML::Builder
+      ###
+      # Convert the builder to HTML
+      def to_html
+        @doc.to_html
+      end
+    end
+  end
+end

data/lib/nokogiri/html/document.rb

@@ -0,0 +1,213 @@
+module Nokogiri
+  module HTML
+    class Document < Nokogiri::XML::Document
+      ###
+      # Get the meta tag encoding for this document.  If there is no meta tag,
+      # then nil is returned.
+      def meta_encoding
+        meta = meta_content_type and
+          /charset\s*=\s*([\w-]+)/i.match(meta['content'])[1]
+      end
+
+      ###
+      # Set the meta tag encoding for this document.  If there is no meta
+      # content tag, the encoding is not set.
+      def meta_encoding= encoding
+        meta = meta_content_type and
+          meta['content'] = "text/html; charset=%s" % encoding
+      end
+
+      def meta_content_type
+        css('meta[@http-equiv]').find { |node|
+          node['http-equiv'] =~ /\AContent-Type\z/i
+        }
+      end
+      private :meta_content_type
+
+      ###
+      # Get the title string of this document.  Return nil if there is
+      # no title tag.
+      def title
+        title = at('title') and title.inner_text
+      end
+
+      ###
+      # Set the title string of this document.  If there is no head
+      # element, the title is not set.
+      def title=(text)
+        unless title = at('title')
+          head = at('head') or return nil
+          title = Nokogiri::XML::Node.new('title', self)
+          head << title
+        end
+        title.children = XML::Text.new(text, self)
+      end
+
+      ####
+      # Serialize Node using +options+.  Save options can also be set using a
+      # block. See SaveOptions.
+      #
+      # These two statements are equivalent:
+      #
+      #  node.serialize(:encoding => 'UTF-8', :save_with => FORMAT | AS_XML)
+      #
+      # or
+      #
+      #   node.serialize(:encoding => 'UTF-8') do |config|
+      #     config.format.as_xml
+      #   end
+      #
+      def serialize options = {}
+        options[:save_with] ||= XML::Node::SaveOptions::DEFAULT_HTML
+        super
+      end
+
+      ####
+      # Create a Nokogiri::XML::DocumentFragment from +tags+
+      def fragment tags = nil
+        DocumentFragment.new(self, tags, self.root)
+      end
+
+      class << self
+        ###
+        # Parse HTML.  +string_or_io+ may be a String, or any object that
+        # responds to _read_ and _close_ such as an IO, or StringIO.
+        # +url+ is resource where this document is located.  +encoding+ is the
+        # encoding that should be used when processing the document. +options+
+        # is a number that sets options in the parser, such as
+        # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
+        # Nokogiri::XML::ParseOptions.
+        def parse string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML
+
+          options = Nokogiri::XML::ParseOptions.new(options) if Fixnum === options
+          # Give the options to the user
+          yield options if block_given?
+
+          if string_or_io.respond_to?(:encoding)
+            unless string_or_io.encoding.name == "ASCII-8BIT"
+              encoding ||= string_or_io.encoding.name
+            end
+          end
+
+          if string_or_io.respond_to?(:read)
+            url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+            if !encoding
+              # Perform advanced encoding detection that libxml2 does
+              # not do.
+              string_or_io = EncodingReader.new(string_or_io)
+              begin
+                return read_io(string_or_io, url, encoding, options.to_i)
+              rescue EncodingFoundException => e
+                # A retry is required because libxml2 has a problem in
+                # that it cannot switch encoding well in the middle of
+                # parsing, especially if it has already seen a
+                # non-ASCII character when it finds an encoding hint.
+                encoding = e.encoding
+              end
+            end
+            return read_io(string_or_io, url, encoding, options.to_i)
+          end
+
+          # read_memory pukes on empty docs
+          return new if string_or_io.nil? or string_or_io.empty?
+
+          if !encoding
+            encoding = EncodingReader.detect_encoding(string_or_io)
+          end
+
+          read_memory(string_or_io, url, encoding, options.to_i)
+        end
+      end
+
+      class EncodingFoundException < Exception # :nodoc:
+        attr_reader :encoding
+
+        def initialize(encoding)
+          @encoding = encoding
+          super("encoding found: %s" % encoding)
+        end
+      end
+
+      class EncodingReader # :nodoc:
+        class SAXHandler < Nokogiri::XML::SAX::Document # :nodoc:
+          attr_reader :encoding
+
+          def found(encoding)
+            @encoding = encoding
+            throw :found
+          end
+
+          def not_found(encoding)
+            found nil
+          end
+
+          def start_element(name, attrs = [])
+            case name
+            when /\A(?:div|h1|img|p|br)\z/
+              not_found
+            when 'meta'
+              attr = Hash[attrs]
+              charset = attr['charset'] and
+                found charset
+              http_equiv = attr['http-equiv'] and
+                http_equiv.match(/\AContent-Type\z/i) and
+                content = attr['content'] and
+                m = content.match(/;\s*charset\s*=\s*([\w-]+)/) and
+                found m[1]
+            end
+          end
+        end
+
+        def self.detect_encoding(chunk)
+          m = chunk.match(/\A(<\?xml[ \t\r\n]+[^>]*>)/) and
+            return Nokogiri.XML(m[1]).encoding
+
+          if Nokogiri.jruby?
+            m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and
+              return m[4]
+          end
+
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML::SAX::Parser.new(handler)
+          catch(:found) {
+            parser.parse(chunk)
+          }
+          handler.encoding
+        rescue
+          nil
+        end
+
+        def initialize(io)
+          @io = io
+          @firstchunk = nil
+        end
+
+        def read(len)
+          # no support for a call without len
+
+          if !@firstchunk
+            @firstchunk = @io.read(len) or return nil
+
+            # This implementation expects that the first call from
+            # htmlReadIO() is made with a length long enough (~1KB) to
+            # achieve advanced encoding detection.
+            if encoding = EncodingReader.detect_encoding(@firstchunk)
+              # The first chunk is stored for the next read in retry.
+              raise EncodingFoundException, encoding
+            end
+          end
+
+          ret = @firstchunk.slice!(0, len)
+          if (len -= ret.length) > 0
+            rest = @io.read(len) and ret << rest
+          end
+          if ret.empty?
+            nil
+          else
+            ret
+          end
+        end
+      end
+    end
+  end
+end