nokogiri 1.15.3 → 1.18.7

data/lib/nokogiri/css/xpath_visitor.rb

@@ -44,6 +44,18 @@ module Nokogiri
         VALUES = [XML, HTML4, HTML5]
       end
 
+      # The visitor configuration set via the +builtins:+ keyword argument to XPathVisitor.new.
+      attr_reader :builtins
+
+      # The visitor configuration set via the +doctype:+ keyword argument to XPathVisitor.new.
+      attr_reader :doctype
+
+      # The visitor configuration set via the +prefix:+ keyword argument to XPathVisitor.new.
+      attr_reader :prefix
+
+      # The visitor configuration set via the +namespaces:+ keyword argument to XPathVisitor.new.
+      attr_reader :namespaces
+
       # :call-seq:
       #   new() → XPathVisitor
       #   new(builtins:, doctype:) → XPathVisitor
@@ -54,7 +66,12 @@ module Nokogiri
       #
       # [Returns] XPathVisitor
       #
-      def initialize(builtins: BuiltinsConfig::NEVER, doctype: DoctypeConfig::XML)
+      def initialize(
+        builtins: BuiltinsConfig::NEVER,
+        doctype: DoctypeConfig::XML,
+        prefix: Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX,
+        namespaces: nil
+      )
         unless BuiltinsConfig::VALUES.include?(builtins)
           raise(ArgumentError, "Invalid values #{builtins.inspect} for builtins: keyword parameter")
         end
@@ -64,6 +81,8 @@ module Nokogiri
 
         @builtins = builtins
         @doctype = doctype
+        @prefix = prefix
+        @namespaces = namespaces
       end
 
       # :call-seq: config() → Hash
@@ -72,7 +91,7 @@ module Nokogiri
       #   a Hash representing the configuration of the XPathVisitor, suitable for use as
       #   part of the CSS cache key.
       def config
-        { builtins: @builtins, doctype: @doctype }
+        { builtins: @builtins, doctype: @doctype, prefix: @prefix, namespaces: @namespaces }
       end
 
       # :stopdoc:
@@ -128,6 +147,8 @@ module Nokogiri
           is_direct = node.value[1].value[0].nil? # e.g. "has(> a)", "has(~ a)", "has(+ a)"
           ".#{"//" unless is_direct}#{node.value[1].accept(self)}"
         else
+          validate_xpath_function_name(node.value.first)
+
           # xpath function call, let's marshal those arguments
           args = ["."]
           args += node.value[1..-1].map do |n|
@@ -207,6 +228,7 @@ module Nokogiri
           when "parent" then "node()"
           when "root" then "not(parent::*)"
           else
+            validate_xpath_function_name(node.value.first)
             "nokogiri:#{node.value.first}(.)"
           end
         end
@@ -255,6 +277,15 @@ module Nokogiri
           else
             "*[local-name()='#{node.value.first}']"
           end
+        elsif node.value.length == 2 # has a namespace prefix
+          if node.value.first.nil? # namespace prefix is empty
+            node.value.last
+          else
+            node.value.join(":")
+          end
+        elsif node.value.first != "*" && @namespaces&.key?("xmlns")
+          # apply the default namespace (if one is present) to a non-wildcard selector
+          "xmlns:#{node.value.first}"
         else
           node.value.first
         end
@@ -270,11 +301,17 @@ module Nokogiri
 
       private
 
+      def validate_xpath_function_name(name)
+        if name.start_with?("-")
+          raise Nokogiri::CSS::SyntaxError, "Invalid XPath function name '#{name}'"
+        end
+      end
+
       def html5_element_name_needs_namespace_handling(node)
-        # if this is the wildcard selector "*", use it as normal
-        node.value.first != "*" &&
-          # if there is already a namespace (i.e., it is a prefixed QName), use it as normal
-          !node.value.first.include?(":")
+        # if there is already a namespace (i.e., it is a prefixed QName), use it as normal
+        node.value.length == 1 &&
+          # if this is the wildcard selector "*", use it as normal
+          node.value.first != "*"
       end
 
       def nth(node, options = {})
@@ -302,7 +339,7 @@ module Nokogiri
       end
 
       def read_a_and_positive_b(values)
-        op = values[2]
+        op = values[2].strip
         if op == "+"
           a = values[0].to_i
           b = values[3].to_i
@@ -335,25 +372,5 @@ module Nokogiri
         end
       end
     end
-
-    module XPathVisitorAlwaysUseBuiltins # :nodoc:
-      def self.new
-        warn(
-          "Nokogiri::CSS::XPathVisitorAlwaysUseBuiltins is deprecated and will be removed in a future version of Nokogiri",
-          { uplevel: 1 },
-        )
-        XPathVisitor.new(builtins: :always)
-      end
-    end
-
-    module XPathVisitorOptimallyUseBuiltins # :nodoc:
-      def self.new
-        warn(
-          "Nokogiri::CSS::XPathVisitorOptimallyUseBuiltins is deprecated and will be removed in a future version of Nokogiri",
-          { uplevel: 1 },
-        )
-        XPathVisitor.new(builtins: :optimal)
-      end
-    end
   end
 end

data/lib/nokogiri/css.rb

@@ -8,53 +8,119 @@ module Nokogiri
       # TODO: Deprecate this method ahead of 2.0 and delete it in 2.0.
       # It is not used by Nokogiri and shouldn't be part of the public API.
       def parse(selector) # :nodoc:
+        warn("Nokogiri::CSS.parse is deprecated and will be removed in a future version of Nokogiri. Use Nokogiri::CSS::Parser#parse instead.", uplevel: 1, category: :deprecated)
         Parser.new.parse(selector)
       end
 
       # :call-seq:
-      #   xpath_for(selector) → String
-      #   xpath_for(selector [, prefix:] [, visitor:] [, ns:]) → String
+      #   xpath_for(selector_list) → Array<String>
+      #   xpath_for(selector_list [, prefix:] [, ns:] [, visitor:] [, cache:]) → Array<String>
       #
-      # Translate a CSS selector to the equivalent XPath query.
+      # Translate a CSS selector list to the equivalent XPath expressions.
+      #
+      # 💡 Note that translated queries are cached by default for performance concerns.
+      #
+      # ⚠ Users should prefer Nokogiri::XML::Searchable#css, which is mixed into all document and
+      # node classes, for querying documents with CSS selectors. This method is the underlying
+      # mechanism used by XML::Searchable and is provided solely for advanced users to translate
+      # \CSS selectors to XPath directly.
+      #
+      # Also see Nokogiri::XML::Searchable#css for documentation on supported CSS selector features,
+      # some extended syntax that Nokogiri supports, and advanced CSS features like pseudo-class
+      # functions.
       #
       # [Parameters]
-      # - +selector+ (String) The CSS selector to be translated into XPath
+      # - +selector_list+ (String)
       #
+      #   The CSS selector to be translated into XPath. This is always a String, but that string
+      #   value may be a {selector list}[https://www.w3.org/TR/selectors-4/#grouping] (see
+      #   examples).
+      #
+      # [Keyword arguments]
       # - +prefix:+ (String)
       #
-      #   The XPath prefix for the query, see Nokogiri::XML::XPath for some options. Default is
-      #   +XML::XPath::GLOBAL_SEARCH_PREFIX+.
+      #   The XPath expression prefix which determines the search context. See Nokogiri::XML::XPath
+      #   for standard options. Default is +XPath::GLOBAL_SEARCH_PREFIX+.
+      #
+      # - +ns:+ (Hash<String ⇒ String>, nil)
+      #
+      #   Namespaces that are referenced in the query, if any. This is a hash where the keys are the
+      #   namespace prefix and the values are the namespace URIs. Default is +nil+ indicating an
+      #   empty set of namespaces.
       #
       # - +visitor:+ (Nokogiri::CSS::XPathVisitor)
       #
-      #   The visitor class to use to transform the AST into XPath. Default is
-      #   +Nokogiri::CSS::XPathVisitor.new+.
+      #   Use this XPathVisitor object to transform the CSS AST into XPath expressions. See
+      #   Nokogiri::CSS::XPathVisitor for more information on some of the complex behavior that can
+      #   be customized for your document type. Default is +Nokogiri::CSS::XPathVisitor.new+.
+      #
+      #   ⚠ Note that this option is mutually exclusive with +prefix+ and +ns+. If +visitor+ is
+      #   provided, +prefix+ and +ns+ must not be present.
+      #
+      # - +cache:+ (Boolean)
+      #
+      #   Whether to use the SelectorCache for the translated query to ensure that repeated queries
+      #   don't incur the overhead of re-parsing the selector. Default is +true+.
       #
-      # - +ns:+ (Hash<String ⇒ String>)
+      # [Returns] (Array<String>) The equivalent set of XPath expressions for +selector_list+
       #
-      #   The namespaces that are referenced in the query, if any. This is a hash where the keys are
-      #   the namespace prefix and the values are the namespace URIs. Default is an empty Hash.
+      # *Example* with a simple selector:
       #
-      # [Returns] (String) The equivalent XPath query for +selector+
+      #   Nokogiri::CSS.xpath_for("div") # => ["//div"]
       #
-      # 💡 Note that translated queries are cached for performance concerns.
+      # *Example* with a compound selector:
       #
-      def xpath_for(selector, options = {})
-        raise TypeError, "no implicit conversion of #{selector.inspect} to String" unless selector.respond_to?(:to_str)
+      #   Nokogiri::CSS.xpath_for("div.xl") # => ["//div[contains(concat(' ',normalize-space(@class),' '),' xl ')]"]
+      #
+      # *Example* with a complex selector:
+      #
+      #   Nokogiri::CSS.xpath_for("h1 + div") # => ["//h1/following-sibling::*[1]/self::div"]
+      #
+      # *Example* with a selector list:
+      #
+      #   Nokogiri::CSS.xpath_for("h1, h2, h3") # => ["//h1", "//h2", "//h3"]
+      #
+      def xpath_for(
+        selector, options = nil,
+        prefix: options&.delete(:prefix),
+        visitor: options&.delete(:visitor),
+        ns: options&.delete(:ns),
+        cache: true
+      )
+        unless options.nil?
+          warn("Nokogiri::CSS.xpath_for: Passing options as an explicit hash is deprecated. Use keyword arguments instead. This will become an error in a future release.", uplevel: 1, category: :deprecated)
+        end
+
+        raise(TypeError, "no implicit conversion of #{selector.inspect} to String") unless selector.respond_to?(:to_str)
 
         selector = selector.to_str
-        raise Nokogiri::CSS::SyntaxError, "empty CSS selector" if selector.empty?
+        raise(Nokogiri::CSS::SyntaxError, "empty CSS selector") if selector.empty?
+
+        if visitor
+          raise ArgumentError, "cannot provide both :prefix and :visitor" if prefix
+          raise ArgumentError, "cannot provide both :ns and :visitor" if ns
+        end
+
+        visitor ||= begin
+          visitor_kw = {}
+          visitor_kw[:prefix] = prefix if prefix
+          visitor_kw[:namespaces] = ns if ns
 
-        prefix = options.fetch(:prefix, Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX)
-        visitor = options.fetch(:visitor) { Nokogiri::CSS::XPathVisitor.new }
-        ns = options.fetch(:ns, {})
+          Nokogiri::CSS::XPathVisitor.new(**visitor_kw)
+        end
 
-        Parser.new(ns).xpath_for(selector, prefix, visitor)
+        if cache
+          key = SelectorCache.key(selector: selector, visitor: visitor)
+          SelectorCache[key] ||= Parser.new.xpath_for(selector, visitor)
+        else
+          Parser.new.xpath_for(selector, visitor)
+        end
       end
     end
   end
 end
 
+require_relative "css/selector_cache"
 require_relative "css/node"
 require_relative "css/xpath_visitor"
 x = $-w

data/lib/nokogiri/decorators/slop.rb

@@ -23,11 +23,9 @@ module Nokogiri
             list = xpath("#{XPATH_PREFIX}#{name}[#{conds}]")
           end
         else
-          CSS::Parser.without_cache do
-            list = xpath(
-              *CSS.xpath_for("#{name}#{args.first}", prefix: XPATH_PREFIX),
-            )
-          end
+          list = xpath(
+            *CSS.xpath_for("#{name}#{args.first}", prefix: XPATH_PREFIX, cache: false),
+          )
         end
 
         super if list.empty?

data/lib/nokogiri/encoding_handler.rb

@@ -6,9 +6,9 @@ module Nokogiri
     # Popular encoding aliases not known by all iconv implementations that Nokogiri should support.
     USEFUL_ALIASES = {
       # alias_name => true_name
-      "NOKOGIRI-SENTINEL" => "UTF-8", # indicating the Nokogiri has installed aliases
+      "ISO-2022-JP" => "ISO-2022-JP", # only for JRuby tests, this is a no-op in CRuby
+      "NOKOGIRI-SENTINEL" => "ISO-2022-JP", # indicating the Nokogiri has installed aliases
       "Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932.
-      "UTF-8" => "UTF-8", # for JRuby tests, this is a no-op in CRuby
     }
 
     class << self

data/lib/nokogiri/html4/document.rb

@@ -92,7 +92,7 @@ module Nokogiri
         title = XML::Node.new("title", self) << tnode
         if (head = at_xpath("//head"))
           head << title
-        elsif (meta = (at_xpath("//meta[@charset]") || meta_content_type))
+        elsif (meta = at_xpath("//meta[@charset]") || meta_content_type)
           # better put after charset declaration
           meta.add_next_sibling(title)
         else
@@ -161,52 +161,73 @@ module Nokogiri
       end
 
       class << self
-        ###
-        # Parse HTML.  +string_or_io+ may be a String, or any object that
-        # responds to _read_ and _close_ such as an IO, or StringIO.
-        # +url+ is resource where this document is located.  +encoding+ is the
-        # encoding that should be used when processing the document. +options+
-        # is a number that sets options in the parser, such as
-        # Nokogiri::XML::ParseOptions::RECOVER.  See the constants in
-        # Nokogiri::XML::ParseOptions.
-        def parse(string_or_io, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML)
+        # :call-seq:
+        #   parse(input) { |options| ... } => Nokogiri::HTML4::Document
+        #   parse(input, url:, encoding:, options:) => Nokogiri::HTML4::Document
+        #
+        # Parse \HTML4 input from a String or IO object, and return a new HTML4::Document.
+        #
+        # [Required Parameters]
+        # - +input+ (String | IO) The content to be parsed.
+        #
+        # [Optional Keyword Arguments]
+        # - +url:+ (String) The base URI for this document.
+        #
+        # - +encoding:+ (String) The name of the encoding that should be used when processing the
+        #   document. When not provided, the encoding will be determined based on the document
+        #   content.
+        #
+        # - +options:+ (Nokogiri::XML::ParseOptions) Configuration object that determines some
+        #   behaviors during parsing. See ParseOptions for more information. The default value is
+        #   +ParseOptions::DEFAULT_HTML+.
+        #
+        # [Yields]
+        #   If a block is given, a Nokogiri::XML::ParseOptions object is yielded to the block which
+        #   can be configured before parsing. See Nokogiri::XML::ParseOptions for more information.
+        #
+        # [Returns] Nokogiri::HTML4::Document
+        def parse(
+          input,
+          url_ = nil, encoding_ = nil, options_ = XML::ParseOptions::DEFAULT_HTML,
+          url: url_, encoding: encoding_, options: options_
+        )
           options = Nokogiri::XML::ParseOptions.new(options) if Integer === options
           yield options if block_given?
 
-          url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
+          url ||= input.respond_to?(:path) ? input.path : nil
 
-          if string_or_io.respond_to?(:encoding)
-            unless string_or_io.encoding == Encoding::ASCII_8BIT
-              encoding ||= string_or_io.encoding.name
+          if input.respond_to?(:encoding)
+            unless input.encoding == Encoding::ASCII_8BIT
+              encoding ||= input.encoding.name
             end
           end
 
-          if string_or_io.respond_to?(:read)
-            if string_or_io.is_a?(Pathname)
+          if input.respond_to?(:read)
+            if input.is_a?(Pathname)
               # resolve the Pathname to the file and open it as an IO object, see #2110
-              string_or_io = string_or_io.expand_path.open
-              url ||= string_or_io.path
+              input = input.expand_path.open
+              url ||= input.path
             end
 
             unless encoding
-              string_or_io = EncodingReader.new(string_or_io)
+              input = EncodingReader.new(input)
               begin
-                return read_io(string_or_io, url, encoding, options.to_i)
+                return read_io(input, url, encoding, options.to_i)
               rescue EncodingReader::EncodingFound => e
                 encoding = e.found_encoding
               end
             end
-            return read_io(string_or_io, url, encoding, options.to_i)
+            return read_io(input, url, encoding, options.to_i)
           end
 
           # read_memory pukes on empty docs
-          if string_or_io.nil? || string_or_io.empty?
+          if input.nil? || input.empty?
             return encoding ? new.tap { |i| i.encoding = encoding } : new
           end
 
-          encoding ||= EncodingReader.detect_encoding(string_or_io)
+          encoding ||= EncodingReader.detect_encoding(input)
 
-          read_memory(string_or_io, url, encoding, options.to_i)
+          read_memory(input, url, encoding, options.to_i)
         end
       end
     end

data/lib/nokogiri/html4/document_fragment.rb

@@ -3,13 +3,83 @@
 module Nokogiri
   module HTML4
     class DocumentFragment < Nokogiri::XML::DocumentFragment
-      ####
-      # Create a Nokogiri::XML::DocumentFragment from +tags+, using +encoding+
-      def self.parse(tags, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block)
+      #
+      # :call-seq:
+      #   parse(input) { |options| ... } → HTML4::DocumentFragment
+      #   parse(input, encoding:, options:) { |options| ... } → HTML4::DocumentFragment
+      #
+      # Parse \HTML4 fragment input from a String, and return a new HTML4::DocumentFragment. This
+      # method creates a new, empty HTML4::Document to contain the fragment.
+      #
+      # [Required Parameters]
+      # - +input+ (String | IO) The content to be parsed.
+      #
+      # [Optional Keyword Arguments]
+      # - +encoding:+ (String) The name of the encoding that should be used when processing the
+      #   document. When not provided, the encoding will be determined based on the document
+      #   content.
+      #
+      # - +options:+ (Nokogiri::XML::ParseOptions) Configuration object that determines some
+      #   behaviors during parsing. See ParseOptions for more information. The default value is
+      #   +ParseOptions::DEFAULT_HTML+.
+      #
+      # [Yields]
+      #   If a block is given, a Nokogiri::XML::ParseOptions object is yielded to the block which
+      #   can be configured before parsing. See ParseOptions for more information.
+      #
+      # [Returns] HTML4::DocumentFragment
+      #
+      # *Example:* Parsing a string
+      #
+      #   fragment = HTML4::DocumentFragment.parse("<div>Hello World</div>")
+      #
+      # *Example:* Parsing an IO
+      #
+      #   fragment = File.open("fragment.html") do |file|
+      #     HTML4::DocumentFragment.parse(file)
+      #   end
+      #
+      # *Example:* Specifying encoding
+      #
+      #   fragment = HTML4::DocumentFragment.parse(input, encoding: "EUC-JP")
+      #
+      # *Example:* Setting parse options dynamically
+      #
+      #   HTML4::DocumentFragment.parse("<div>Hello World") do |options|
+      #     options.huge.pedantic
+      #   end
+      #
+      def self.parse(
+        input,
+        encoding_ = nil, options_ = XML::ParseOptions::DEFAULT_HTML,
+        encoding: encoding_, options: options_,
+        &block
+      )
+        # TODO: this method should take a context node.
         doc = HTML4::Document.new
 
-        encoding ||= if tags.respond_to?(:encoding)
-          encoding = tags.encoding
+        if input.respond_to?(:read)
+          # Handle IO-like objects (IO, File, StringIO, etc.)
+          # The _read_ method of these objects doesn't accept an +encoding+ parameter.
+          # Encoding is usually set when the IO object is created or opened,
+          # or by using the _set_encoding_ method.
+          #
+          # 1. If +encoding+ is provided and the object supports _set_encoding_,
+          #    set the encoding before reading.
+          # 2. Read the content from the IO-like object.
+          #
+          # Note: After reading, the content's encoding will be:
+          # - The encoding set by _set_encoding_ if it was called
+          # - The default encoding of the IO object otherwise
+          #
+          # For StringIO specifically, _set_encoding_ affects only the internal string,
+          # not how the data is read out.
+          input.set_encoding(encoding) if encoding && input.respond_to?(:set_encoding)
+          input = input.read
+        end
+
+        encoding ||= if input.respond_to?(:encoding)
+          encoding = input.encoding
           if encoding == ::Encoding::ASCII_8BIT
             "UTF-8"
           else
@@ -21,29 +91,71 @@ module Nokogiri
 
         doc.encoding = encoding
 
-        new(doc, tags, nil, options, &block)
+        new(doc, input, options: options, &block)
       end
 
-      def initialize(document, tags = nil, ctx = nil, options = XML::ParseOptions::DEFAULT_HTML) # rubocop:disable Lint/MissingSuper
-        return self unless tags
+      #
+      # :call-seq:
+      #   new(document) { |options| ... } → HTML4::DocumentFragment
+      #   new(document, input) { |options| ... } → HTML4::DocumentFragment
+      #   new(document, input, context:, options:) { |options| ... } → HTML4::DocumentFragment
+      #
+      # Parse \HTML4 fragment input from a String, and return a new HTML4::DocumentFragment.
+      #
+      # 💡 It's recommended to use either HTML4::DocumentFragment.parse or XML::Node#parse rather
+      # than call this method directly.
+      #
+      # [Required Parameters]
+      # - +document+ (HTML4::Document) The parent document to associate the returned fragment with.
+      #
+      # [Optional Parameters]
+      # - +input+ (String) The content to be parsed.
+      #
+      # [Optional Keyword Arguments]
+      # - +context:+ (Nokogiri::XML::Node) The <b>context node</b> for the subtree created. See
+      #   below for more information.
+      #
+      # - +options:+ (Nokogiri::XML::ParseOptions) Configuration object that determines some
+      #   behaviors during parsing. See ParseOptions for more information. The default value is
+      #   +ParseOptions::DEFAULT_HTML+.
+      #
+      # [Yields]
+      #   If a block is given, a Nokogiri::XML::ParseOptions object is yielded to the block which
+      #   can be configured before parsing. See ParseOptions for more information.
+      #
+      # [Returns] HTML4::DocumentFragment
+      #
+      # === Context \Node
+      #
+      # If a context node is specified using +context:+, then the fragment will be created by
+      # calling XML::Node#parse on that node, so the parser will behave as if that Node is the
+      # parent of the fragment subtree.
+      #
+      def initialize(
+        document, input = nil,
+        context_ = nil, options_ = XML::ParseOptions::DEFAULT_HTML,
+        context: context_, options: options_
+      ) # rubocop:disable Lint/MissingSuper
+        return self unless input
 
         options = Nokogiri::XML::ParseOptions.new(options) if Integer === options
+        @parse_options = options
         yield options if block_given?
 
-        if ctx
+        if context
           preexisting_errors = document.errors.dup
-          node_set = ctx.parse("<div>#{tags}</div>", options)
+          node_set = context.parse("<div>#{input}</div>", options)
           node_set.first.children.each { |child| child.parent = self } unless node_set.empty?
           self.errors = document.errors - preexisting_errors
         else
           # This is a horrible hack, but I don't care
-          path = if /^\s*?<body/i.match?(tags)
+          path = if /^\s*?<body/i.match?(input)
             "/html/body"
           else
             "/html/body/node()"
           end
 
-          temp_doc = HTML4::Document.parse("<html><body>#{tags}", nil, document.encoding, options)
+          temp_doc = HTML4::Document.parse("<html><body>#{input}", nil, document.encoding, options)
           temp_doc.xpath(path).each { |child| child.parent = self }
           self.errors = temp_doc.errors
         end

data/lib/nokogiri/html4/encoding_reader.rb

@@ -26,7 +26,7 @@ module Nokogiri
 
         def initialize
           @encoding = nil
-          super()
+          super
         end
 
         def start_element(name, attrs = [])
@@ -94,7 +94,7 @@ module Nokogiri
         # no support for a call without len
 
         unless @firstchunk
-          (@firstchunk = @io.read(len)) || (return nil)
+          (@firstchunk = @io.read(len)) || return
 
           # This implementation expects that the first call from
           # htmlReadIO() is made with a length long enough (~1KB) to