nokogiri 1.13.9-x86_64-linux → 1.14.0.rc1-x86_64-linux

data/gumbo-parser/Makefile

@@ -13,8 +13,18 @@ LDFLAGS := -pthread
 
 all: check
 
+# don't try to regenerate ragel or gperf files in CI, that should be a development-only action and
+# the generated files should be committed to SCM
+ifneq ($(CI),true)
+src/foreign_attrs.c: src/foreign_attrs.gperf
+	gperf -m100 -n $< | ./gperf-filter.sed > $@
+
+src/%.c: src/%.gperf
+	gperf -m100 $< | ./gperf-filter.sed > $@
+
 src/%.c: src/%.rl
 	ragel -F1 -o $@ $<
+endif
 
 build/src:
 	mkdir -p $@

data/lib/nokogiri/css/node.rb

@@ -40,9 +40,9 @@ module Nokogiri
 
       # Convert to_type
       def to_type
-        [@type] + @value.map do |n|
+        [@type] + @value.filter_map do |n|
           n.to_type if n.respond_to?(:to_type)
-        end.compact
+        end
       end
 
       # Convert to array

data/lib/nokogiri/css/xpath_visitor.rb

@@ -278,7 +278,9 @@ module Nokogiri
       end
 
       def nth(node, options = {})
-        raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
+        unless node.value.size == 4
+          raise(ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}")
+        end
 
         a, b = read_a_and_positive_b(node.value)
         position = if options[:child]

data/lib/nokogiri/css.rb

@@ -40,9 +40,15 @@ module Nokogiri
       # 💡 Note that translated queries are cached for performance concerns.
       #
       def xpath_for(selector, options = {})
+        raise TypeError, "no implicit conversion of #{selector.inspect} to String" unless selector.respond_to?(:to_str)
+
+        selector = selector.to_str
+        raise Nokogiri::CSS::SyntaxError, "empty CSS selector" if selector.empty?
+
         prefix = options.fetch(:prefix, Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX)
         visitor = options.fetch(:visitor) { Nokogiri::CSS::XPathVisitor.new }
         ns = options.fetch(:ns, {})
+
         Parser.new(ns).xpath_for(selector, prefix, visitor)
       end
     end

data/lib/nokogiri/encoding_handler.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Nokogiri
+  class EncodingHandler
+    # Popular encoding aliases not known by all iconv implementations that Nokogiri should support.
+    USEFUL_ALIASES = {
+      # alias_name => true_name
+      "NOKOGIRI-SENTINEL" => "UTF-8", # indicating the Nokogiri has installed aliases
+      "Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932.
+      "UTF-8" => "UTF-8", # for JRuby tests, this is a no-op in CRuby
+    }
+
+    class << self
+      def install_default_aliases
+        USEFUL_ALIASES.each do |alias_name, name|
+          EncodingHandler.alias(name, alias_name) if EncodingHandler[alias_name].nil?
+        end
+      end
+    end
+
+    # :stopdoc:
+    if Nokogiri.jruby?
+      class << self
+        def [](name)
+          storage.key?(name) ? new(storage[name]) : nil
+        end
+
+        def alias(name, alias_name)
+          storage[alias_name] = name
+        end
+
+        def delete(name)
+          storage.delete(name)
+        end
+
+        def clear_aliases!
+          storage.clear
+        end
+
+        private
+
+        def storage
+          @storage ||= {}
+        end
+      end
+
+      def initialize(name)
+        @name = name
+      end
+
+      attr_reader :name
+    end
+  end
+end
+
+Nokogiri::EncodingHandler.install_default_aliases

data/lib/nokogiri/extension.rb

@@ -3,13 +3,14 @@
 # load the C or Java extension
 begin
   # native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version>
-  ::RUBY_VERSION =~ /(\d+\.\d+)/
+  RUBY_VERSION =~ /(\d+\.\d+)/
   require_relative "#{Regexp.last_match(1)}/nokogiri"
 rescue LoadError => e
   if /GLIBC/.match?(e.message)
     warn(<<~EOM)
 
-      ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system with glibc < 2.17:
+      ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system
+             with an unsupported version of glibc.
 
         #{e.message}
 

data/lib/nokogiri/html4/document.rb

@@ -176,7 +176,7 @@ module Nokogiri
           url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
 
           if string_or_io.respond_to?(:encoding)
-            unless string_or_io.encoding.name == "ASCII-8BIT"
+            unless string_or_io.encoding == Encoding::ASCII_8BIT
               encoding ||= string_or_io.encoding.name
             end
           end
@@ -189,21 +189,10 @@ module Nokogiri
             end
 
             unless encoding
-              # Libxml2's parser has poor support for encoding
-              # detection.  First, it does not recognize the HTML5
-              # style meta charset declaration.  Secondly, even if it
-              # successfully detects an encoding hint, it does not
-              # re-decode or re-parse the preceding part which may be
-              # garbled.
-              #
-              # EncodingReader aims to perform advanced encoding
-              # detection beyond what Libxml2 does, and to emulate
-              # rewinding of a stream and make Libxml2 redo parsing
-              # from the start when an encoding hint is found.
               string_or_io = EncodingReader.new(string_or_io)
               begin
                 return read_io(string_or_io, url, encoding, options.to_i)
-              rescue EncodingFound => e
+              rescue EncodingReader::EncodingFound => e
                 encoding = e.found_encoding
               end
             end
@@ -220,114 +209,6 @@ module Nokogiri
           read_memory(string_or_io, url, encoding, options.to_i)
         end
       end
-
-      class EncodingFound < StandardError # :nodoc: all
-        attr_reader :found_encoding
-
-        def initialize(encoding)
-          @found_encoding = encoding
-          super(format("encoding found: %s", encoding))
-        end
-      end
-
-      # :nodoc: all
-      class EncodingReader
-        class SAXHandler < Nokogiri::XML::SAX::Document
-          attr_reader :encoding
-
-          def initialize
-            @encoding = nil
-            super()
-          end
-
-          def start_element(name, attrs = [])
-            return unless name == "meta"
-
-            attr = Hash[attrs]
-            (charset = attr["charset"]) &&
-              (@encoding = charset)
-            (http_equiv = attr["http-equiv"]) &&
-              http_equiv.match(/\AContent-Type\z/i) &&
-              (content = attr["content"]) &&
-              (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
-              (@encoding = m[1])
-          end
-        end
-
-        class JumpSAXHandler < SAXHandler
-          def initialize(jumptag)
-            @jumptag = jumptag
-            super()
-          end
-
-          def start_element(name, attrs = [])
-            super
-            throw(@jumptag, @encoding) if @encoding
-            throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
-          end
-        end
-
-        def self.detect_encoding(chunk)
-          (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
-            (return Nokogiri.XML(m[1]).encoding)
-
-          if Nokogiri.jruby?
-            (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
-              (return m[4])
-            catch(:encoding_found) do
-              Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
-              nil
-            end
-          else
-            handler = SAXHandler.new
-            parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
-            begin
-              parser << chunk
-            rescue
-              Nokogiri::SyntaxError
-            end
-            handler.encoding
-          end
-        end
-
-        def initialize(io)
-          @io = io
-          @firstchunk = nil
-          @encoding_found = nil
-        end
-
-        # This method is used by the C extension so that
-        # Nokogiri::HTML4::Document#read_io() does not leak memory when
-        # EncodingFound is raised.
-        attr_reader :encoding_found
-
-        def read(len)
-          # no support for a call without len
-
-          unless @firstchunk
-            (@firstchunk = @io.read(len)) || (return nil)
-
-            # This implementation expects that the first call from
-            # htmlReadIO() is made with a length long enough (~1KB) to
-            # achieve advanced encoding detection.
-            if (encoding = EncodingReader.detect_encoding(@firstchunk))
-              # The first chunk is stored for the next read in retry.
-              raise @encoding_found = EncodingFound.new(encoding)
-            end
-          end
-          @encoding_found = nil
-
-          ret = @firstchunk.slice!(0, len)
-          if (len -= ret.length) > 0
-            (rest = @io.read(len)) && ret << (rest)
-          end
-          if ret.empty?
-            nil
-          else
-            ret
-          end
-        end
-      end
     end
   end
 end

data/lib/nokogiri/html4/element_description_defaults.rb

@@ -25,43 +25,37 @@ module Nokogiri
 
       unless method_defined?(:implied_start_tag?)
         def implied_start_tag?
-          d = default_desc
-          d ? d.startTag : nil
+          default_desc&.startTag
         end
       end
 
       unless method_defined?(:implied_end_tag?)
         def implied_end_tag?
-          d = default_desc
-          d ? d.endTag : nil
+          default_desc&.endTag
         end
       end
 
       unless method_defined?(:save_end_tag?)
         def save_end_tag?
-          d = default_desc
-          d ? d.saveEndTag : nil
+          default_desc&.saveEndTag
         end
       end
 
       unless method_defined?(:deprecated?)
         def deprecated?
-          d = default_desc
-          d ? d.depr : nil
+          default_desc&.depr
         end
       end
 
       unless method_defined?(:description)
         def description
-          d = default_desc
-          d ? d.desc : nil
+          default_desc&.desc
         end
       end
 
       unless method_defined?(:default_sub_element)
         def default_sub_element
-          d = default_desc
-          d ? d.defaultsubelt : nil
+          default_desc&.defaultsubelt
         end
       end
 

data/lib/nokogiri/html4/encoding_reader.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Nokogiri
+  module HTML4
+    # Libxml2's parser has poor support for encoding detection.  First, it does not recognize the
+    # HTML5 style meta charset declaration.  Secondly, even if it successfully detects an encoding
+    # hint, it does not re-decode or re-parse the preceding part which may be garbled.
+    #
+    # EncodingReader aims to perform advanced encoding detection beyond what Libxml2 does, and to
+    # emulate rewinding of a stream and make Libxml2 redo parsing from the start when an encoding
+    # hint is found.
+
+    # :nodoc: all
+    class EncodingReader
+      class EncodingFound < StandardError
+        attr_reader :found_encoding
+
+        def initialize(encoding)
+          @found_encoding = encoding
+          super(format("encoding found: %s", encoding))
+        end
+      end
+
+      class SAXHandler < Nokogiri::XML::SAX::Document
+        attr_reader :encoding
+
+        def initialize
+          @encoding = nil
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          return unless name == "meta"
+
+          attr = Hash[attrs]
+          (charset = attr["charset"]) &&
+            (@encoding = charset)
+          (http_equiv = attr["http-equiv"]) &&
+            http_equiv.match(/\AContent-Type\z/i) &&
+            (content = attr["content"]) &&
+            (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
+            (@encoding = m[1])
+        end
+      end
+
+      class JumpSAXHandler < SAXHandler
+        def initialize(jumptag)
+          @jumptag = jumptag
+          super()
+        end
+
+        def start_element(name, attrs = [])
+          super
+          throw(@jumptag, @encoding) if @encoding
+          throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
+        end
+      end
+
+      def self.detect_encoding(chunk)
+        (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
+          (return Nokogiri.XML(m[1]).encoding)
+
+        if Nokogiri.jruby?
+          (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
+            (return m[4])
+          catch(:encoding_found) do
+            Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
+            nil
+          end
+        else
+          handler = SAXHandler.new
+          parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
+          begin
+            parser << chunk
+          rescue
+            Nokogiri::SyntaxError
+          end
+          handler.encoding
+        end
+      end
+
+      def initialize(io)
+        @io = io
+        @firstchunk = nil
+        @encoding_found = nil
+      end
+
+      # This method is used by the C extension so that
+      # Nokogiri::HTML4::Document#read_io() does not leak memory when
+      # EncodingFound is raised.
+      attr_reader :encoding_found
+
+      def read(len)
+        # no support for a call without len
+
+        unless @firstchunk
+          (@firstchunk = @io.read(len)) || (return nil)
+
+          # This implementation expects that the first call from
+          # htmlReadIO() is made with a length long enough (~1KB) to
+          # achieve advanced encoding detection.
+          if (encoding = EncodingReader.detect_encoding(@firstchunk))
+            # The first chunk is stored for the next read in retry.
+            raise @encoding_found = EncodingFound.new(encoding)
+          end
+        end
+        @encoding_found = nil
+
+        ret = @firstchunk.slice!(0, len)
+        if (len -= ret.length) > 0
+          (rest = @io.read(len)) && ret << (rest)
+        end
+        if ret.empty?
+          nil
+        else
+          ret
+        end
+      end
+    end
+  end
+end

data/lib/nokogiri/html4.rb

@@ -39,6 +39,7 @@ end
 require_relative "html4/entity_lookup"
 require_relative "html4/document"
 require_relative "html4/document_fragment"
+require_relative "html4/encoding_reader"
 require_relative "html4/sax/parser_context"
 require_relative "html4/sax/parser"
 require_relative "html4/sax/push_parser"

data/lib/nokogiri/html5/document.rb

@@ -21,48 +21,137 @@ require_relative "../html4/document"
 
 module Nokogiri
   module HTML5
+    # Enum for the HTML5 parser quirks mode values. Values returned by HTML5::Document#quirks_mode
+    #
+    # See https://dom.spec.whatwg.org/#concept-document-quirks for more information on HTML5 quirks
+    # mode.
+    #
+    # Since v1.14.0
+    module QuirksMode
+      NO_QUIRKS = 0 # The document was parsed in "no-quirks" mode
+      QUIRKS = 1 # The document was parsed in "quirks" mode
+      LIMITED_QUIRKS = 2 # The document was parsed in "limited-quirks" mode
+    end
+
     # Since v1.12.0
     #
     # 💡 HTML5 functionality is not available when running JRuby.
     class Document < Nokogiri::HTML4::Document
-      def self.parse(string_or_io, url = nil, encoding = nil, **options, &block)
-        yield options if block
-        string_or_io = "" unless string_or_io
+      # Get the url name for this document, as passed into Document.parse, Document.read_io, or
+      # Document.read_memory
+      attr_reader :url
 
-        if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != "ASCII-8BIT"
-          encoding ||= string_or_io.encoding.name
-        end
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::Document.new`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
 
-        if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
-          url ||= string_or_io.path
+      class << self
+        # :call-seq:
+        #   parse(input)
+        #   parse(input, url=nil, encoding=nil, **options)
+        #   parse(input, url=nil, encoding=nil) { |options| ... }
+        #
+        # Parse HTML5 input.
+        #
+        # [Parameters]
+        # - +input+ may be a String, or any object that responds to _read_ and _close_ such as an
+        #   IO, or StringIO.
+        #
+        # - +url+ (optional) is a String indicating the canonical URI where this document is located.
+        #
+        # - +encoding+ (optional) is the encoding that should be used when processing
+        #   the document.
+        #
+        # - +options+ (optional) is a configuration Hash (or keyword arguments) to set options
+        #   during parsing. The three currently supported options are +:max_errors+,
+        #   +:max_tree_depth+ and +:max_attributes+, described at Nokogiri::HTML5.
+        #
+        #   ⚠ Note that these options are different than those made available by
+        #   Nokogiri::XML::Document and Nokogiri::HTML4::Document.
+        #
+        # - +block+ (optional) is passed a configuration Hash on which parse options may be set. See
+        #   Nokogiri::HTML5 for more information and usage.
+        #
+        # [Returns] Nokogiri::HTML5::Document
+        #
+        def parse(string_or_io, url = nil, encoding = nil, **options, &block)
+          yield options if block
+          string_or_io = "" unless string_or_io
+
+          if string_or_io.respond_to?(:encoding) && string_or_io.encoding != Encoding::ASCII_8BIT
+            encoding ||= string_or_io.encoding.name
+          end
+
+          if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
+            url ||= string_or_io.path
+          end
+          unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+            raise ArgumentError, "not a string or IO object"
+          end
+
+          do_parse(string_or_io, url, encoding, options)
         end
-        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
-          raise ArgumentError, "not a string or IO object"
+
+        # Create a new document from an IO object.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_io(io, url = nil, encoding = nil, **options)
+          raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+
+          do_parse(io, url, encoding, options)
         end
 
-        do_parse(string_or_io, url, encoding, options)
-      end
+        # Create a new document from a String.
+        #
+        # 💡 Most users should prefer Document.parse to this method.
+        def read_memory(string, url = nil, encoding = nil, **options)
+          raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
 
-      def self.read_io(io, url = nil, encoding = nil, **options)
-        raise ArgumentError, "io object doesn't respond to :read" unless io.respond_to?(:read)
+          do_parse(string, url, encoding, options)
+        end
 
-        do_parse(io, url, encoding, options)
-      end
+        private
 
-      def self.read_memory(string, url = nil, encoding = nil, **options)
-        raise ArgumentError, "string object doesn't respond to :to_str" unless string.respond_to?(:to_str)
+        def do_parse(string_or_io, url, encoding, options)
+          string = HTML5.read_and_encode(string_or_io, encoding)
+          max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
+          max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
+          max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
+          doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth, self)
+          doc.encoding = "UTF-8"
+          doc
+        end
+      end
 
-        do_parse(string, url, encoding, options)
+      def initialize(*args) # :nodoc:
+        super
+        @url = nil
+        @quirks_mode = nil
       end
 
-      def fragment(tags = nil)
-        DocumentFragment.new(self, tags, root)
+      # :call-seq:
+      #   fragment() → Nokogiri::HTML5::DocumentFragment
+      #   fragment(markup) → Nokogiri::HTML5::DocumentFragment
+      #
+      # Parse a HTML5 document fragment from +markup+, returning a Nokogiri::HTML5::DocumentFragment.
+      #
+      # [Properties]
+      # - +markup+ (String) The HTML5 markup fragment to be parsed
+      #
+      # [Returns]
+      #   Nokogiri::HTML5::DocumentFragment. This object's children will be empty if `markup` is not passed, is empty, or is `nil`.
+      #
+      def fragment(markup = nil)
+        DocumentFragment.new(self, markup)
       end
 
-      def to_xml(options = {}, &block)
+      def to_xml(options = {}, &block) # :nodoc:
         # Bypass XML::Document#to_xml which doesn't add
         # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does.
-        XML::Node.instance_method(:to_xml).bind(self).call(options, &block)
+        XML::Node.instance_method(:to_xml).bind_call(self, options, &block)
       end
 
       # :call-seq:
@@ -70,22 +159,10 @@ module Nokogiri
       #
       # [Returns] The document type which determines CSS-to-XPath translation.
       #
-      # See XPathVisitor for more information.
+      # See CSS::XPathVisitor for more information.
       def xpath_doctype
         Nokogiri::CSS::XPathVisitor::DoctypeConfig::HTML5
       end
-
-      private
-
-      def self.do_parse(string_or_io, url, encoding, options)
-        string = HTML5.read_and_encode(string_or_io, encoding)
-        max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES
-        max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS
-        max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth)
-        doc.encoding = "UTF-8"
-        doc
-      end
     end
   end
 end

data/lib/nokogiri/html5/document_fragment.rb

@@ -28,6 +28,13 @@ module Nokogiri
       attr_accessor :document
       attr_accessor :errors
 
+      # Get the parser's quirks mode value. See HTML5::QuirksMode.
+      #
+      # This method returns `nil` if the parser was not invoked (e.g., `Nokogiri::HTML5::DocumentFragment.new(doc)`).
+      #
+      # Since v1.14.0
+      attr_reader :quirks_mode
+
       # Create a document fragment.
       def initialize(doc, tags = nil, ctx = nil, options = {})
         self.document = doc
@@ -41,10 +48,10 @@ module Nokogiri
         Nokogiri::Gumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
 
-      def serialize(options = {}, &block)
+      def serialize(options = {}, &block) # :nodoc:
         # Bypass XML::Document.serialize which doesn't support options even
         # though XML::Node.serialize does!
-        XML::Node.instance_method(:serialize).bind(self).call(options, &block)
+        XML::Node.instance_method(:serialize).bind_call(self, options, &block)
       end
 
       # Parse a document fragment from +tags+, returning a Nodeset.