nokogiri 1.13.10 → 1.14.0.rc1

data/gumbo-parser/Makefile

@@ -13,8 +13,18 @@ LDFLAGS := -pthread
 
 all: check
 
+# don't try to regenerate ragel or gperf files in CI, that should be a development-only action and
+# the generated files should be committed to SCM
+ifneq ($(CI),true)
+src/foreign_attrs.c: src/foreign_attrs.gperf
+	gperf -m100 -n $< | ./gperf-filter.sed > $@
+
+src/%.c: src/%.gperf
+	gperf -m100 $< | ./gperf-filter.sed > $@
+
 src/%.c: src/%.rl
 	ragel -F1 -o $@ $<
+endif
 
 build/src:
 	mkdir -p $@

data/gumbo-parser/src/attribute.h

@@ -1,7 +1,7 @@
 #ifndef GUMBO_ATTRIBUTE_H_
 #define GUMBO_ATTRIBUTE_H_
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 #ifdef __cplusplus
 extern "C" {

data/gumbo-parser/src/error.c

@@ -21,7 +21,7 @@
 #include <string.h>
 #include "ascii.h"
 #include "error.h"
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "macros.h"
 #include "parser.h"
 #include "string_buffer.h"

data/gumbo-parser/src/error.h

@@ -3,7 +3,7 @@
 
 #include <stdint.h>
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "insertion_mode.h"
 #include "string_buffer.h"
 #include "token_type.h"

data/gumbo-parser/src/foreign_attrs.c

@@ -1,7 +1,7 @@
 /* ANSI-C code produced by gperf version 3.1 */
-/* Command-line: gperf -m100 -n lib/foreign_attrs.gperf  */
+/* Command-line: gperf -m100 -n src/foreign_attrs.gperf  */
 /* Computed positions: -k'2,8' */
-/* Filtered by: mk/gperf-filter.sed */
+/* Filtered by: gperf-filter.sed */
 
 #include "replacement.h"
 #include "macros.h"

data/gumbo-parser/src/{gumbo.h → nokogiri_gumbo.h}

@@ -292,6 +292,7 @@ typedef enum {
   GUMBO_TAG_TT,
   GUMBO_TAG_RTC,
   GUMBO_TAG_DIALOG,
+  GUMBO_TAG_SEARCH,
   // Used for all tags that don't have special handling in HTML.
   GUMBO_TAG_UNKNOWN,
   // A marker value to indicate the end of the enum, for iterating over it.

data/gumbo-parser/src/parser.c

@@ -24,7 +24,7 @@
 #include "ascii.h"
 #include "attribute.h"
 #include "error.h"
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "insertion_mode.h"
 #include "macros.h"
 #include "parser.h"
@@ -2940,7 +2940,7 @@ static void handle_in_body(GumboParser* parser, GumboToken* token) {
       TAG(DETAILS), TAG(DIALOG), TAG(DIR), TAG(DIV), TAG(DL), TAG(FIELDSET),
       TAG(FIGCAPTION), TAG(FIGURE), TAG(FOOTER), TAG(HEADER), TAG(HGROUP),
       TAG(MAIN), TAG(MENU), TAG(NAV), TAG(OL), TAG(P), TAG(SECTION),
-      TAG(SUMMARY), TAG(UL)
+      TAG(SUMMARY), TAG(UL), TAG(SEARCH)
     })
   ) {
     maybe_implicitly_close_p_tag(parser, token);
@@ -3018,7 +3018,7 @@ static void handle_in_body(GumboParser* parser, GumboToken* token) {
       TAG(CENTER), TAG(DETAILS), TAG(DIALOG), TAG(DIR), TAG(DIV), TAG(DL),
       TAG(FIELDSET), TAG(FIGCAPTION), TAG(FIGURE), TAG(FOOTER), TAG(HEADER),
       TAG(HGROUP), TAG(LISTING), TAG(MAIN), TAG(MENU), TAG(NAV), TAG(OL),
-      TAG(PRE), TAG(SECTION), TAG(SUMMARY), TAG(UL)
+      TAG(PRE), TAG(SECTION), TAG(SUMMARY), TAG(UL), TAG(SEARCH)
     })
   ) {
     GumboTag tag = token->v.end_tag.tag;
@@ -3057,6 +3057,9 @@ static void handle_in_body(GumboParser* parser, GumboToken* token) {
         ignore_token(parser);
         return;
       }
+      // Since we remove the form node without popping, we need to make sure
+      // that we flush any text nodes at the end of the form.
+      maybe_flush_text_node_buffer(parser);
       // This differs from implicitly_close_tags because we remove *only* the
       // <form> element; other nodes are left in scope.
       generate_implied_end_tags(parser, GUMBO_TAG_LAST, NULL);
@@ -3434,7 +3437,7 @@ static void handle_in_table(GumboParser* parser, GumboToken* token) {
      || token->type == GUMBO_TOKEN_WHITESPACE
      || token->type == GUMBO_TOKEN_NULL)
     && node_tag_in_set(get_current_node(parser), &(const TagSet) {
-      TAG(TABLE), TAG(TBODY), TAG(TFOOT), TAG(THEAD), TAG(TR)
+      TAG(TABLE), TAG(TBODY), TAG(TEMPLATE), TAG(TFOOT), TAG(THEAD), TAG(TR)
     })
   ) {
     // The "pending table character tokens" list described in the spec is

data/gumbo-parser/src/replacement.h

@@ -2,7 +2,7 @@
 #define GUMBO_REPLACEMENT_H_
 
 #include <stddef.h>
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 typedef struct {
   const char *const from;

data/gumbo-parser/src/string_buffer.h

@@ -4,7 +4,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 #ifdef __cplusplus
 extern "C" {

data/gumbo-parser/src/string_piece.c

@@ -17,7 +17,7 @@
 
 #include <stddef.h>
 #include <string.h>
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "ascii.h"
 
 bool gumbo_string_equals (

data/gumbo-parser/src/svg_attrs.c

@@ -1,7 +1,7 @@
 /* ANSI-C code produced by gperf version 3.1 */
-/* Command-line: gperf -m100 lib/svg_attrs.gperf  */
+/* Command-line: gperf -m100 src/svg_attrs.gperf  */
 /* Computed positions: -k'1,10,$' */
-/* Filtered by: mk/gperf-filter.sed */
+/* Filtered by: gperf-filter.sed */
 
 #include "replacement.h"
 #include "macros.h"

data/gumbo-parser/src/svg_tags.c

@@ -1,7 +1,7 @@
 /* ANSI-C code produced by gperf version 3.1 */
-/* Command-line: gperf -m100 lib/svg_tags.gperf  */
+/* Command-line: gperf -m100 src/svg_tags.gperf  */
 /* Computed positions: -k'3,7' */
-/* Filtered by: mk/gperf-filter.sed */
+/* Filtered by: gperf-filter.sed */
 
 #include "replacement.h"
 #include "macros.h"

data/gumbo-parser/src/tag.c

@@ -14,7 +14,7 @@
  limitations under the License.
 */
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "util.h"
 #include "tag_lookup.h"
 
@@ -172,6 +172,7 @@ static const char kGumboTagNames[GUMBO_TAG_LAST+1][15] = {
     [GUMBO_TAG_TT] = "tt",
     [GUMBO_TAG_RTC] = "rtc",
     [GUMBO_TAG_DIALOG] = "dialog",
+    [GUMBO_TAG_SEARCH] = "search",
 
     [GUMBO_TAG_UNKNOWN] = "",
     [GUMBO_TAG_LAST] = "",

data/gumbo-parser/src/tag_lookup.c

@@ -1,14 +1,14 @@
 /* ANSI-C code produced by gperf version 3.1 */
-/* Command-line: gperf -m100 lib/tag_lookup.gperf  */
+/* Command-line: gperf -m100 src/tag_lookup.gperf  */
 /* Computed positions: -k'1-2,$' */
-/* Filtered by: mk/gperf-filter.sed */
+/* Filtered by: gperf-filter.sed */
 
 #include "tag_lookup.h"
 #include "macros.h"
 #include "ascii.h"
 #include <string.h>
 
-#define TOTAL_KEYWORDS 150
+#define TOTAL_KEYWORDS 151
 #define MIN_WORD_LENGTH 1
 #define MAX_WORD_LENGTH 14
 #define MIN_HASH_VALUE 9
@@ -26,7 +26,7 @@ hash (register const char *str, register size_t len)
       272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
       272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
       272, 272, 272, 272, 272, 272, 272, 272, 272, 272,
-      272, 272, 272, 272, 272, 272, 272, 272, 272,   9,
+      272, 272, 272, 272, 272, 272, 272, 272, 272,  11,
         7,   6,   4,   4,   3,   4,   3,   3, 272, 272,
       272, 272, 272, 272, 272,  70,  83, 152,   7,  16,
        61,  98,   5,  76, 102, 126,  12,  19,  54,  54,
@@ -69,7 +69,7 @@ gumbo_tag_lookup (register const char *str, register size_t len)
   static const unsigned char lengthtable[] =
     {
        0,  0,  0,  0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  2,
-       2,  2,  2,  6,  2,  6,  2,  4,  0,  7,  6,  3,  0,  3,
+       2,  2,  2,  6,  2,  6,  6,  4,  2,  7,  6,  3,  0,  3,
        0,  6,  6,  8,  5,  0,  0,  4,  5,  5,  8,  0,  2,  4,
        5,  2,  0,  5,  4,  2,  0,  7,  0,  8,  5,  0,  0,  0,
        0,  0,  0,  5,  3,  4,  5,  1,  4,  0,  4,  1,  2,  8,
@@ -111,9 +111,9 @@ gumbo_tag_lookup (register const char *str, register size_t len)
       {"spacer", GUMBO_TAG_SPACER},
       {"h2", GUMBO_TAG_H2},
       {"header", GUMBO_TAG_HEADER},
-      {"h1", GUMBO_TAG_H1},
+      {"search", GUMBO_TAG_SEARCH},
       {"head", GUMBO_TAG_HEAD},
-      {(char*)0,GUMBO_TAG_UNKNOWN},
+      {"h1", GUMBO_TAG_H1},
       {"details", GUMBO_TAG_DETAILS},
       {"select", GUMBO_TAG_SELECT},
       {"dir", GUMBO_TAG_DIR},

data/gumbo-parser/src/tag_lookup.gperf

@@ -167,3 +167,4 @@ spacer, GUMBO_TAG_SPACER
 tt, GUMBO_TAG_TT
 rtc, GUMBO_TAG_RTC
 dialog, GUMBO_TAG_DIALOG
+search, GUMBO_TAG_SEARCH

data/gumbo-parser/src/tag_lookup.h

@@ -1,7 +1,7 @@
 #ifndef GUMBO_TAG_LOOKUP_H_
 #define GUMBO_TAG_LOOKUP_H_
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 typedef struct {
     const char *key;

data/gumbo-parser/src/token_buffer.h

@@ -20,7 +20,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 #ifdef __cplusplus
 extern "C" {

data/gumbo-parser/src/tokenizer.c

@@ -50,7 +50,7 @@
 #include "attribute.h"
 #include "char_ref.h"
 #include "error.h"
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "parser.h"
 #include "string_buffer.h"
 #include "token_type.h"

data/gumbo-parser/src/tokenizer.h

@@ -7,7 +7,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "token_type.h"
 #include "tokenizer_states.h"
 

data/gumbo-parser/src/utf8.c

@@ -22,7 +22,7 @@
 #include <string.h>
 
 #include "error.h"
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "parser.h"
 #include "ascii.h"
 #include "vector.h"

data/gumbo-parser/src/utf8.h

@@ -19,7 +19,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 #include "macros.h"
 
 #ifdef __cplusplus

data/gumbo-parser/src/util.c

@@ -19,7 +19,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include "util.h"
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 void* gumbo_alloc(size_t size) {
   void* ptr = malloc(size);
@@ -63,6 +63,4 @@ void gumbo_debug(const char* format, ...) {
   va_end(args);
   fflush(stdout);
 }
-#else
-void gumbo_debug(const char* UNUSED_ARG(format), ...) {}
 #endif

data/gumbo-parser/src/util.h

@@ -21,7 +21,11 @@ void* gumbo_realloc(void* ptr, size_t size) RETURNS_NONNULL;
 void gumbo_free(void* ptr);
 
 // Debug wrapper for printf
+#ifdef GUMBO_DEBUG
 void gumbo_debug(const char* format, ...) PRINTF(1);
+#else
+static inline void PRINTF(1) gumbo_debug(const char* UNUSED_ARG(format), ...) {};
+#endif
 
 #ifdef __cplusplus
 }

data/gumbo-parser/src/vector.h

@@ -1,7 +1,7 @@
 #ifndef GUMBO_VECTOR_H_
 #define GUMBO_VECTOR_H_
 
-#include "gumbo.h"
+#include "nokogiri_gumbo.h"
 
 #ifdef __cplusplus
 extern "C" {

data/lib/nokogiri/css/node.rb

@@ -40,9 +40,9 @@ module Nokogiri
 
       # Convert to_type
       def to_type
-        [@type] + @value.map do |n|
+        [@type] + @value.filter_map do |n|
           n.to_type if n.respond_to?(:to_type)
-        end.compact
+        end
       end
 
       # Convert to array

data/lib/nokogiri/css/xpath_visitor.rb

@@ -278,7 +278,9 @@ module Nokogiri
       end
 
       def nth(node, options = {})
-        raise ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}" unless node.value.size == 4
+        unless node.value.size == 4
+          raise(ArgumentError, "expected an+b node to contain 4 tokens, but is #{node.value.inspect}")
+        end
 
         a, b = read_a_and_positive_b(node.value)
         position = if options[:child]

data/lib/nokogiri/css.rb

@@ -40,9 +40,15 @@ module Nokogiri
       # 💡 Note that translated queries are cached for performance concerns.
       #
       def xpath_for(selector, options = {})
+        raise TypeError, "no implicit conversion of #{selector.inspect} to String" unless selector.respond_to?(:to_str)
+
+        selector = selector.to_str
+        raise Nokogiri::CSS::SyntaxError, "empty CSS selector" if selector.empty?
+
         prefix = options.fetch(:prefix, Nokogiri::XML::XPath::GLOBAL_SEARCH_PREFIX)
         visitor = options.fetch(:visitor) { Nokogiri::CSS::XPathVisitor.new }
         ns = options.fetch(:ns, {})
+
         Parser.new(ns).xpath_for(selector, prefix, visitor)
       end
     end

data/lib/nokogiri/encoding_handler.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Nokogiri
+  class EncodingHandler
+    # Popular encoding aliases not known by all iconv implementations that Nokogiri should support.
+    USEFUL_ALIASES = {
+      # alias_name => true_name
+      "NOKOGIRI-SENTINEL" => "UTF-8", # indicating the Nokogiri has installed aliases
+      "Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932.
+      "UTF-8" => "UTF-8", # for JRuby tests, this is a no-op in CRuby
+    }
+
+    class << self
+      def install_default_aliases
+        USEFUL_ALIASES.each do |alias_name, name|
+          EncodingHandler.alias(name, alias_name) if EncodingHandler[alias_name].nil?
+        end
+      end
+    end
+
+    # :stopdoc:
+    if Nokogiri.jruby?
+      class << self
+        def [](name)
+          storage.key?(name) ? new(storage[name]) : nil
+        end
+
+        def alias(name, alias_name)
+          storage[alias_name] = name
+        end
+
+        def delete(name)
+          storage.delete(name)
+        end
+
+        def clear_aliases!
+          storage.clear
+        end
+
+        private
+
+        def storage
+          @storage ||= {}
+        end
+      end
+
+      def initialize(name)
+        @name = name
+      end
+
+      attr_reader :name
+    end
+  end
+end
+
+Nokogiri::EncodingHandler.install_default_aliases

data/lib/nokogiri/extension.rb

@@ -3,13 +3,14 @@
 # load the C or Java extension
 begin
   # native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version>
-  ::RUBY_VERSION =~ /(\d+\.\d+)/
+  RUBY_VERSION =~ /(\d+\.\d+)/
   require_relative "#{Regexp.last_match(1)}/nokogiri"
 rescue LoadError => e
   if /GLIBC/.match?(e.message)
     warn(<<~EOM)
 
-      ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system with glibc < 2.17:
+      ERROR: It looks like you're trying to use Nokogiri as a precompiled native gem on a system
+             with an unsupported version of glibc.
 
         #{e.message}
 

data/lib/nokogiri/html4/document.rb

@@ -176,7 +176,7 @@ module Nokogiri
           url ||= string_or_io.respond_to?(:path) ? string_or_io.path : nil
 
           if string_or_io.respond_to?(:encoding)
-            unless string_or_io.encoding.name == "ASCII-8BIT"
+            unless string_or_io.encoding == Encoding::ASCII_8BIT
               encoding ||= string_or_io.encoding.name
             end
           end
@@ -189,21 +189,10 @@ module Nokogiri
             end
 
             unless encoding
-              # Libxml2's parser has poor support for encoding
-              # detection.  First, it does not recognize the HTML5
-              # style meta charset declaration.  Secondly, even if it
-              # successfully detects an encoding hint, it does not
-              # re-decode or re-parse the preceding part which may be
-              # garbled.
-              #
-              # EncodingReader aims to perform advanced encoding
-              # detection beyond what Libxml2 does, and to emulate
-              # rewinding of a stream and make Libxml2 redo parsing
-              # from the start when an encoding hint is found.
               string_or_io = EncodingReader.new(string_or_io)
               begin
                 return read_io(string_or_io, url, encoding, options.to_i)
-              rescue EncodingFound => e
+              rescue EncodingReader::EncodingFound => e
                 encoding = e.found_encoding
               end
             end
@@ -220,114 +209,6 @@ module Nokogiri
           read_memory(string_or_io, url, encoding, options.to_i)
         end
       end
-
-      class EncodingFound < StandardError # :nodoc: all
-        attr_reader :found_encoding
-
-        def initialize(encoding)
-          @found_encoding = encoding
-          super(format("encoding found: %s", encoding))
-        end
-      end
-
-      # :nodoc: all
-      class EncodingReader
-        class SAXHandler < Nokogiri::XML::SAX::Document
-          attr_reader :encoding
-
-          def initialize
-            @encoding = nil
-            super()
-          end
-
-          def start_element(name, attrs = [])
-            return unless name == "meta"
-
-            attr = Hash[attrs]
-            (charset = attr["charset"]) &&
-              (@encoding = charset)
-            (http_equiv = attr["http-equiv"]) &&
-              http_equiv.match(/\AContent-Type\z/i) &&
-              (content = attr["content"]) &&
-              (m = content.match(/;\s*charset\s*=\s*([\w-]+)/)) &&
-              (@encoding = m[1])
-          end
-        end
-
-        class JumpSAXHandler < SAXHandler
-          def initialize(jumptag)
-            @jumptag = jumptag
-            super()
-          end
-
-          def start_element(name, attrs = [])
-            super
-            throw(@jumptag, @encoding) if @encoding
-            throw(@jumptag, nil) if /\A(?:div|h1|img|p|br)\z/.match?(name)
-          end
-        end
-
-        def self.detect_encoding(chunk)
-          (m = chunk.match(/\A(<\?xml[ \t\r\n][^>]*>)/)) &&
-            (return Nokogiri.XML(m[1]).encoding)
-
-          if Nokogiri.jruby?
-            (m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i)) &&
-              (return m[4])
-            catch(:encoding_found) do
-              Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk)
-              nil
-            end
-          else
-            handler = SAXHandler.new
-            parser = Nokogiri::HTML4::SAX::PushParser.new(handler)
-            begin
-              parser << chunk
-            rescue
-              Nokogiri::SyntaxError
-            end
-            handler.encoding
-          end
-        end
-
-        def initialize(io)
-          @io = io
-          @firstchunk = nil
-          @encoding_found = nil
-        end
-
-        # This method is used by the C extension so that
-        # Nokogiri::HTML4::Document#read_io() does not leak memory when
-        # EncodingFound is raised.
-        attr_reader :encoding_found
-
-        def read(len)
-          # no support for a call without len
-
-          unless @firstchunk
-            (@firstchunk = @io.read(len)) || (return nil)
-
-            # This implementation expects that the first call from
-            # htmlReadIO() is made with a length long enough (~1KB) to
-            # achieve advanced encoding detection.
-            if (encoding = EncodingReader.detect_encoding(@firstchunk))
-              # The first chunk is stored for the next read in retry.
-              raise @encoding_found = EncodingFound.new(encoding)
-            end
-          end
-          @encoding_found = nil
-
-          ret = @firstchunk.slice!(0, len)
-          if (len -= ret.length) > 0
-            (rest = @io.read(len)) && ret << (rest)
-          end
-          if ret.empty?
-            nil
-          else
-            ret
-          end
-        end
-      end
     end
   end
 end

data/lib/nokogiri/html4/element_description_defaults.rb

@@ -25,43 +25,37 @@ module Nokogiri
 
       unless method_defined?(:implied_start_tag?)
         def implied_start_tag?
-          d = default_desc
-          d ? d.startTag : nil
+          default_desc&.startTag
         end
       end
 
       unless method_defined?(:implied_end_tag?)
         def implied_end_tag?
-          d = default_desc
-          d ? d.endTag : nil
+          default_desc&.endTag
         end
       end
 
       unless method_defined?(:save_end_tag?)
         def save_end_tag?
-          d = default_desc
-          d ? d.saveEndTag : nil
+          default_desc&.saveEndTag
         end
       end
 
       unless method_defined?(:deprecated?)
         def deprecated?
-          d = default_desc
-          d ? d.depr : nil
+          default_desc&.depr
         end
       end
 
       unless method_defined?(:description)
         def description
-          d = default_desc
-          d ? d.desc : nil
+          default_desc&.desc
         end
       end
 
       unless method_defined?(:default_sub_element)
         def default_sub_element
-          d = default_desc
-          d ? d.defaultsubelt : nil
+          default_desc&.defaultsubelt
         end
       end