RubyGems - nokogumbo - Versions diffs - 2.0.2 → 2.0.3 - Mend

nokogumbo 2.0.2 → 2.0.3

Files changed (14) hide show

checksums.yaml +4 -4
data/README.md +18 -2
data/ext/nokogumbo/nokogumbo.c +49 -11
data/gumbo-parser/src/gumbo.h +19 -0
data/gumbo-parser/src/parser.c +3 -0
data/gumbo-parser/src/tokenizer.c +11 -1
data/gumbo-parser/src/utf8.c +5 -0
data/gumbo-parser/src/utf8.h +1 -0
data/lib/nokogumbo.rb +3 -0
data/lib/nokogumbo/html5.rb +15 -14
data/lib/nokogumbo/html5/document.rb +7 -2
data/lib/nokogumbo/html5/document_fragment.rb +2 -1
data/lib/nokogumbo/version.rb +1 -1
metadata +6 -6

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbc23d3b7a9665b48d0516f523756407ca46733286e89edaf2b5b01b05820ffd
-  data.tar.gz: 9d777f65fe5170fe66fde53dd9fa0d2904e2ca2d73d25d6fd240fc383cb20804
+  metadata.gz: e4694cf3eefbeee2a55cd4bb355b7ec6159c64eac4454dff02b1fbf7e5e8375a
+  data.tar.gz: 67832a7c26148f59755360758fcc0b0c1969949bf1e5a1b27f5cabe4b9e8b40b
 SHA512:
-  metadata.gz: 602ecadc3afd998eb380a9d4fed204cdd7fd48e5ee8539e4ae5713289fa65a923d27c4b67835546c048522f1a04a165d87f257f1389c591e41e127be06e98109
-  data.tar.gz: 88a714a552e5cc6f11b65da15bd9816d21bfc383dd5c68e1b6e8ab82ce842adf3159ad56579428b87b7c1dba002b9e38758494865153285f2563aeae9edeba54
+  metadata.gz: 3a415817caaf0c3c03037664bda8ed8aa17cc14419e75672dcaa2e2a7dd6d9a20e6ab59095a2295f90da5e45de2c3d72f9a25557533836d55dc67966fe8c7a14
+  data.tar.gz: 8dc8f9f2d55936a63097301dc5eb6fb54ed1e4c274b03cdcd6f45e2b4ac2cdc911a54e8e5838ce468820ebade731a62f7cfc167817528fd0adb415087ce924b6

data/README.md CHANGED

@@ -1,7 +1,7 @@
 # Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
-Nokogumbo provides the ability for a Ruby program to invoke the
-[Gumbo HTML5 parser](https://github.com/google/gumbo-parser#readme)
+Nokogumbo provides the ability for a Ruby program to invoke
+[our version of the  Gumbo HTML5 parser](https://github.com/rubys/nokogumbo/tree/master/gumbo-parser/src)
 and to access the result as a
 [Nokogiri::HTML::Document](http://rdoc.info/github/sparklemotion/nokogiri/Nokogiri/HTML/Document).
@@ -128,6 +128,22 @@ doc = Nokogiri.HTML5(html)
 doc = Nokogiri.HTML5(html, max_tree_depth: -1)
 ```
+### Attribute limit per element
+The maximum number of attributes per DOM element is configurable by the
+`:max_attributes` option. If a given element would exceed this limit, then an
+[ArgumentError](https://ruby-doc.org/core-2.5.0/ArgumentError.html) is thrown.
+This limit (which defaults to `Nokogumbo::DEFAULT_MAX_ATTRIBUTES = 400`) can
+be removed by giving the option `max_attributes: -1`.
+``` ruby
+html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
+# "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
+doc = Nokogiri.HTML5(html)
+# raises ArgumentError: Attributes per element limit exceeded
+doc = Nokogiri.HTML5(html, max_attributes: -1)
+```
 ## HTML Serialization
 After parsing HTML, it may be serialized using any of the Nokogiri

data/ext/nokogumbo/nokogumbo.c CHANGED

@@ -281,6 +281,7 @@ static GumboOutput *perform_parse(const GumboOptions *options, VALUE input) {
   switch (output->status) {
   case GUMBO_STATUS_OK:
     break;
+  case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
   case GUMBO_STATUS_TREE_TOO_DEEP:
     gumbo_destroy_output(output);
     rb_raise(rb_eArgError, "%s", status_string);
@@ -479,19 +480,43 @@ typedef struct {
   xmlDocPtr doc;
 } ParseArgs;
-static VALUE parse_cleanup(ParseArgs *args) {
+static void parse_args_mark(void *parse_args) {
+  ParseArgs *args = parse_args;
+  rb_gc_mark_maybe(args->input);
+  rb_gc_mark_maybe(args->url_or_frag);
+}
+// Wrap a ParseArgs pointer. The underlying ParseArgs must outlive the
+// wrapper.
+static VALUE wrap_parse_args(ParseArgs *args) {
+  return Data_Wrap_Struct(rb_cData, parse_args_mark, RUBY_NEVER_FREE, args);
+}
+// Returnsd the underlying ParseArgs wrapped by wrap_parse_args.
+static ParseArgs *unwrap_parse_args(VALUE obj) {
+  ParseArgs *args;
+  Data_Get_Struct(obj, ParseArgs, args);
+  return args;
+}
+static VALUE parse_cleanup(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   gumbo_destroy_output(args->output);
+  // Make sure garbage collection doesn't mark the objects as being live based
+  // on references from the ParseArgs. This may be unnecessary.
+  args->input = Qnil;
+  args->url_or_frag = Qnil;
   if (args->doc != NIL)
     xmlFreeDoc(args->doc);
   return Qnil;
 }
-static VALUE parse_continue(ParseArgs *args);
+static VALUE parse_continue(VALUE parse_args);
 // Parse a string using gumbo_parse into a Nokogiri document
-static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE max_depth) {
+static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_attributes, VALUE max_errors, VALUE max_depth) {
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   options.max_tree_depth = NUM2INT(max_depth);
@@ -502,10 +527,13 @@ static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE m
     .url_or_frag = url,
     .doc = NIL,
   };
-  return rb_ensure(parse_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  return rb_ensure(parse_continue, parse_args, parse_cleanup, parse_args);
 }
-static VALUE parse_continue(ParseArgs *args) {
+static VALUE parse_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   xmlDocPtr doc;
   if (output->document->v.document.has_doctype) {
@@ -563,13 +591,14 @@ static xmlNodePtr extract_xml_node(VALUE node) {
 #endif
 }
-static VALUE fragment_continue(ParseArgs *args);
+static VALUE fragment_continue(VALUE parse_args);
 static VALUE fragment (
   VALUE self,
   VALUE doc_fragment,
   VALUE tags,
   VALUE ctx,
+  VALUE max_attributes,
   VALUE max_errors,
   VALUE max_depth
 ) {
@@ -676,6 +705,7 @@ static VALUE fragment (
   // Perform a fragment parse.
   int depth = NUM2INT(max_depth);
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   // Add one to account for the HTML element.
   options.max_tree_depth = depth < 0 ? -1 : (depth + 1);
@@ -692,11 +722,13 @@ static VALUE fragment (
     .url_or_frag = doc_fragment,
     .doc = (xmlDocPtr)extract_xml_node(doc),
   };
-  rb_ensure(fragment_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  rb_ensure(fragment_continue, parse_args, parse_cleanup, parse_args);
   return Qnil;
 }
-static VALUE fragment_continue(ParseArgs *args) {
+static VALUE fragment_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   VALUE doc_fragment = args->url_or_frag;
   xmlDocPtr xml_doc = args->doc;
@@ -720,10 +752,15 @@ void Init_nokogumbo() {
   VALUE mNokogiri = rb_const_get(rb_cObject, rb_intern_const("Nokogiri"));
   VALUE mNokogiriXml = rb_const_get(mNokogiri, rb_intern_const("XML"));
   cNokogiriXmlSyntaxError = rb_const_get(mNokogiriXml, rb_intern_const("SyntaxError"));
+  rb_gc_register_mark_object(cNokogiriXmlSyntaxError);
   cNokogiriXmlElement = rb_const_get(mNokogiriXml, rb_intern_const("Element"));
+  rb_gc_register_mark_object(cNokogiriXmlElement);
   cNokogiriXmlText = rb_const_get(mNokogiriXml, rb_intern_const("Text"));
+  rb_gc_register_mark_object(cNokogiriXmlText);
   cNokogiriXmlCData = rb_const_get(mNokogiriXml, rb_intern_const("CDATA"));
+  rb_gc_register_mark_object(cNokogiriXmlCData);
   cNokogiriXmlComment = rb_const_get(mNokogiriXml, rb_intern_const("Comment"));
+  rb_gc_register_mark_object(cNokogiriXmlComment);
   // Interned symbols.
   new = rb_intern_const("new");
@@ -736,6 +773,7 @@ void Init_nokogumbo() {
   // Class constants.
   VALUE HTML5 = rb_const_get(mNokogiri, rb_intern_const("HTML5"));
   Document = rb_const_get(HTML5, rb_intern_const("Document"));
+  rb_gc_register_mark_object(Document);
   // Interned symbols.
   internal_subset = rb_intern_const("internal_subset");
@@ -743,8 +781,8 @@ void Init_nokogumbo() {
   // Define Nokogumbo module with parse and fragment methods.
   VALUE Gumbo = rb_define_module("Nokogumbo");
-  rb_define_singleton_method(Gumbo, "parse", parse, 4);
-  rb_define_singleton_method(Gumbo, "fragment", fragment, 5);
+  rb_define_singleton_method(Gumbo, "parse", parse, 5);
+  rb_define_singleton_method(Gumbo, "fragment", fragment, 6);
   // Add private constant for testing.
   rb_define_const(Gumbo, "LINE_SUPPORTED", line_supported);

data/gumbo-parser/src/gumbo.h CHANGED

@@ -706,6 +706,15 @@ typedef struct GumboInternalOptions {
    */
   bool stop_on_first_error;
+  /**
+   * Maximum allowed number of attributes per element. If this limit is
+   * exceeded, the parser will return early with a partial document and
+   * the returned `GumboOutput` will have its `status` field set to
+   * `GUMBO_STATUS_TOO_MANY_ATTRIBUTES`. Set to `-1` to disable the limit.
+   * Default: `400`.
+   */
+  int max_attributes;
   /**
    * Maximum allowed depth for the parse tree. If this limit is exceeded,
    * the parser will return early with a partial document and the returned
@@ -796,6 +805,16 @@ typedef enum {
    */
   GUMBO_STATUS_TREE_TOO_DEEP,
+  /**
+   * Indicates that the maximum number of attributes per element
+   * (`GumboOptions::max_attributes`) was reached during parsing. The
+   * resulting tree will be a partial document, with no further nodes
+   * created after the point where the limit was reached. The partial
+   * document may be useful for constructing an error message but
+   * typically shouldn't be used for other purposes.
+   */
+  GUMBO_STATUS_TOO_MANY_ATTRIBUTES,
   // Currently unused
   GUMBO_STATUS_OUT_OF_MEMORY,
 } GumboOutputStatus;

data/gumbo-parser/src/parser.c CHANGED

@@ -48,6 +48,7 @@ typedef uint8_t TagSet[GUMBO_TAG_LAST + 1];
 const GumboOptions kGumboDefaultOptions = {
   .tab_stop = 8,
   .stop_on_first_error = false,
+  .max_attributes = 400,
   .max_tree_depth = 400,
   .max_errors = -1,
   .fragment_context = NULL,
@@ -4858,6 +4859,8 @@ const char* gumbo_status_to_string(GumboOutputStatus status) {
       return "OK";
     case GUMBO_STATUS_OUT_OF_MEMORY:
       return "System allocator returned NULL during parsing";
+    case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
+      return "Attributes per element limit exceeded";
     case GUMBO_STATUS_TREE_TOO_DEEP:
       return "Document tree depth limit exceeded";
     default:

data/gumbo-parser/src/tokenizer.c CHANGED

@@ -784,12 +784,22 @@ static void add_duplicate_attr_error(GumboParser* parser) {
 static void finish_attribute_name(GumboParser* parser) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   GumboTagState* tag_state = &tokenizer->_tag_state;
+  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
+  int max_attributes = parser->_options->max_attributes;
+  if (unlikely(max_attributes >= 0 && attributes->length >= (unsigned int) max_attributes)) {
+    parser->_output->status = GUMBO_STATUS_TOO_MANY_ATTRIBUTES;
+    gumbo_debug("Attributes limit exceeded.\n");
+    reinitialize_tag_buffer(parser);
+    tag_state->_drop_next_attr_value = true;
+    return;
+  }
   // May've been set by a previous attribute without a value; reset it here.
   tag_state->_drop_next_attr_value = false;
   assert(tag_state->_attributes.data);
   assert(tag_state->_attributes.capacity);
-  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
   for (unsigned int i = 0; i < attributes->length; ++i) {
     GumboAttribute* attr = attributes->data[i];
     if (

data/gumbo-parser/src/utf8.c CHANGED

@@ -193,6 +193,11 @@ void utf8iterator_init (
   iter->_pos.offset = 0;
   iter->_parser = parser;
   read_char(iter);
+  if (iter->_current == kUtf8BomChar) {
+    iter->_start += iter->_width;
+    iter->_pos.offset += iter->_width;
+    read_char(iter);
+  }
 }
 void utf8iterator_next(Utf8Iterator* iter) {

data/gumbo-parser/src/utf8.h CHANGED

@@ -31,6 +31,7 @@ struct GumboInternalParser;
 // Unicode replacement char.
 #define kUtf8ReplacementChar 0xFFFD
+#define kUtf8BomChar 0xFEFF
 #define kUtf8MaxChar 0x10FFFF
 typedef struct GumboInternalUtf8Iterator {

data/lib/nokogumbo.rb CHANGED

@@ -5,6 +5,9 @@ require 'nokogumbo/html5'
 require 'nokogumbo/nokogumbo'
 module Nokogumbo
+  # The default maximum number of attributes per element.
+  DEFAULT_MAX_ATTRIBUTES = 400
   # The default maximum number of errors for parsing a document or a fragment.
   DEFAULT_MAX_ERRORS = 0

data/lib/nokogumbo/html5.rb CHANGED

@@ -19,7 +19,7 @@ module Nokogiri
     # Parse an HTML 5 document. Convenience method for Nokogiri::HTML5::Document.parse
     def self.parse(string, url = nil, encoding = nil, **options, &block)
-      Document.parse(string, url, encoding, options, &block)
+      Document.parse(string, url, encoding, **options, &block)
     end
     # Parse a fragment from +string+. Convenience method for
@@ -92,19 +92,20 @@ module Nokogiri
         if encoding.nil?
           string = string.read
         else
-        string = string.read(encoding: encoding)
+          string = string.read(encoding: encoding)
         end
       else
         # Otherwise the string has the given encoding.
-        if encoding && string.respond_to?(:force_encoding)
+        string = string.to_str
+        if encoding
           string = string.dup
           string.force_encoding(encoding)
         end
       end
-      # convert to UTF-8 (Ruby 1.9+)
-      if string.respond_to?(:encoding) && string.encoding != Encoding::UTF_8
-        string = reencode(string.dup)
+      # convert to UTF-8
+      if string.encoding != Encoding::UTF_8
+        string = reencode(string)
       end
       string
     end
@@ -123,18 +124,17 @@ module Nokogiri
     # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
     #
     def self.reencode(body, content_type=nil)
-      return body unless body.respond_to? :encoding
       if body.encoding == Encoding::ASCII_8BIT
         encoding = nil
         # look for a Byte Order Mark (BOM)
-        if body[0..1] == "\xFE\xFF"
-          encoding = 'utf-16be'
-        elsif body[0..1] == "\xFF\xFE"
-          encoding = 'utf-16le'
-        elsif body[0..2] == "\xEF\xBB\xBF"
-          encoding = 'utf-8'
+        initial_bytes = body[0..2].bytes
+        if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
+          encoding = Encoding::UTF_8
+        elsif initial_bytes[0..1] == [0xFE, 0xFF]
+          encoding = Encoding::UTF_16BE
+        elsif initial_bytes[0..1] == [0xFF, 0xFE]
+          encoding = Encoding::UTF_16LE
         end
         # look for a charset in a content-encoding header
@@ -154,6 +154,7 @@ module Nokogiri
         encoding ||= Encoding::ISO_8859_1
         # change the encoding to match the detected or inferred encoding
+        body = body.dup
         begin
           body.force_encoding(encoding)
         rescue ArgumentError

data/lib/nokogumbo/html5/document.rb CHANGED

@@ -12,6 +12,9 @@ module Nokogiri
         if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
           url ||= string_or_io.path
         end
+        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+          raise ArgumentError.new("not a string or IO object")
+        end
         do_parse(string_or_io, url, encoding, options)
       end
@@ -21,7 +24,8 @@ module Nokogiri
       end
       def self.read_memory(string, url = nil, encoding = nil, **options)
-        do_parse(string.to_s, url, encoding, options)
+        raise ArgumentError.new("string object doesn't respond to :to_str") unless string.respond_to?(:to_str)
+        do_parse(string, url, encoding, options)
       end
       def fragment(tags = nil)
@@ -37,9 +41,10 @@ module Nokogiri
       private
       def self.do_parse(string_or_io, url, encoding, options)
         string = HTML5.read_and_encode(string_or_io, encoding)
+        max_attributes = options[:max_attributes] || Nokogumbo::DEFAULT_MAX_ATTRIBUTES
         max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogumbo::DEFAULT_MAX_ERRORS
         max_depth = options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogumbo.parse(string.to_s, url, max_errors, max_depth)
+        doc = Nokogumbo.parse(string, url, max_attributes, max_errors, max_depth)
         doc.encoding = 'UTF-8'
         doc
       end

data/lib/nokogumbo/html5/document_fragment.rb CHANGED

@@ -12,10 +12,11 @@ module Nokogiri
         self.errors = []
         return self unless tags
+        max_attributes = options[:max_attributes] || Nokogumbo::DEFAULT_MAX_ATTRIBUTES
         max_errors = options[:max_errors] || Nokogumbo::DEFAULT_MAX_ERRORS
         max_depth = options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH
         tags = Nokogiri::HTML5.read_and_encode(tags, nil)
-        Nokogumbo.fragment(self, tags, ctx, max_errors, max_depth)
+        Nokogumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
       def serialize(options = {}, &block)

data/lib/nokogumbo/version.rb CHANGED

@@ -1,3 +1,3 @@
 module Nokogumbo
-  VERSION = "2.0.2"
+  VERSION = "2.0.3"
 end

metadata CHANGED

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: nokogumbo
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Sam Ruby
 - Stephen Checkoway
-autorequire:
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2020-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -94,7 +94,7 @@ metadata:
   changelog_uri: https://github.com/rubys/nokogumbo/blob/master/CHANGELOG.md
   homepage_uri: https://github.com/rubys/nokogumbo/#readme
   source_code_uri: https://github.com/rubys/nokogumbo
-post_install_message:
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,8 +109,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key:
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Nokogiri interface to the Gumbo HTML5 parser
 test_files: []