RubyGems - nokogumbo - Versions diffs - 2.0.2 → 2.0.3 - Mend

nokogumbo 2.0.2 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

checksums.yaml +4 -4
data/README.md +18 -2
data/ext/nokogumbo/nokogumbo.c +49 -11
data/gumbo-parser/src/gumbo.h +19 -0
data/gumbo-parser/src/parser.c +3 -0
data/gumbo-parser/src/tokenizer.c +11 -1
data/gumbo-parser/src/utf8.c +5 -0
data/gumbo-parser/src/utf8.h +1 -0
data/lib/nokogumbo.rb +3 -0
data/lib/nokogumbo/html5.rb +15 -14
data/lib/nokogumbo/html5/document.rb +7 -2
data/lib/nokogumbo/html5/document_fragment.rb +2 -1
data/lib/nokogumbo/version.rb +1 -1
metadata +6 -6

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbc23d3b7a9665b48d0516f523756407ca46733286e89edaf2b5b01b05820ffd
-  data.tar.gz: 9d777f65fe5170fe66fde53dd9fa0d2904e2ca2d73d25d6fd240fc383cb20804
+  metadata.gz: e4694cf3eefbeee2a55cd4bb355b7ec6159c64eac4454dff02b1fbf7e5e8375a
+  data.tar.gz: 67832a7c26148f59755360758fcc0b0c1969949bf1e5a1b27f5cabe4b9e8b40b
 SHA512:
-  metadata.gz: 602ecadc3afd998eb380a9d4fed204cdd7fd48e5ee8539e4ae5713289fa65a923d27c4b67835546c048522f1a04a165d87f257f1389c591e41e127be06e98109
-  data.tar.gz: 88a714a552e5cc6f11b65da15bd9816d21bfc383dd5c68e1b6e8ab82ce842adf3159ad56579428b87b7c1dba002b9e38758494865153285f2563aeae9edeba54
+  metadata.gz: 3a415817caaf0c3c03037664bda8ed8aa17cc14419e75672dcaa2e2a7dd6d9a20e6ab59095a2295f90da5e45de2c3d72f9a25557533836d55dc67966fe8c7a14
+  data.tar.gz: 8dc8f9f2d55936a63097301dc5eb6fb54ed1e4c274b03cdcd6f45e2b4ac2cdc911a54e8e5838ce468820ebade731a62f7cfc167817528fd0adb415087ce924b6

data/README.md CHANGED

@@ -1,7 +1,7 @@
 # Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
-Nokogumbo provides the ability for a Ruby program to invoke the
-[Gumbo HTML5 parser](https://github.com/google/gumbo-parser#readme)
+Nokogumbo provides the ability for a Ruby program to invoke
+[our version of the  Gumbo HTML5 parser](https://github.com/rubys/nokogumbo/tree/master/gumbo-parser/src)
 and to access the result as a
 [Nokogiri::HTML::Document](http://rdoc.info/github/sparklemotion/nokogiri/Nokogiri/HTML/Document).
@@ -128,6 +128,22 @@ doc = Nokogiri.HTML5(html)
 doc = Nokogiri.HTML5(html, max_tree_depth: -1)
 ```
+### Attribute limit per element
+The maximum number of attributes per DOM element is configurable by the
+`:max_attributes` option. If a given element would exceed this limit, then an
+[ArgumentError](https://ruby-doc.org/core-2.5.0/ArgumentError.html) is thrown.
+This limit (which defaults to `Nokogumbo::DEFAULT_MAX_ATTRIBUTES = 400`) can
+be removed by giving the option `max_attributes: -1`.
+``` ruby
+html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
+# "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
+doc = Nokogiri.HTML5(html)
+# raises ArgumentError: Attributes per element limit exceeded
+doc = Nokogiri.HTML5(html, max_attributes: -1)
+```
 ## HTML Serialization
 After parsing HTML, it may be serialized using any of the Nokogiri

data/ext/nokogumbo/nokogumbo.c CHANGED

@@ -281,6 +281,7 @@ static GumboOutput *perform_parse(const GumboOptions *options, VALUE input) {
   switch (output->status) {
   case GUMBO_STATUS_OK:
     break;
+  case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
   case GUMBO_STATUS_TREE_TOO_DEEP:
     gumbo_destroy_output(output);
     rb_raise(rb_eArgError, "%s", status_string);
@@ -479,19 +480,43 @@ typedef struct {
   xmlDocPtr doc;
 } ParseArgs;
-static VALUE parse_cleanup(ParseArgs *args) {
+static void parse_args_mark(void *parse_args) {
+  ParseArgs *args = parse_args;
+  rb_gc_mark_maybe(args->input);
+  rb_gc_mark_maybe(args->url_or_frag);
+}
+// Wrap a ParseArgs pointer. The underlying ParseArgs must outlive the
+// wrapper.
+static VALUE wrap_parse_args(ParseArgs *args) {
+  return Data_Wrap_Struct(rb_cData, parse_args_mark, RUBY_NEVER_FREE, args);
+}
+// Returnsd the underlying ParseArgs wrapped by wrap_parse_args.
+static ParseArgs *unwrap_parse_args(VALUE obj) {
+  ParseArgs *args;
+  Data_Get_Struct(obj, ParseArgs, args);
+  return args;
+}
+static VALUE parse_cleanup(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   gumbo_destroy_output(args->output);
+  // Make sure garbage collection doesn't mark the objects as being live based
+  // on references from the ParseArgs. This may be unnecessary.
+  args->input = Qnil;
+  args->url_or_frag = Qnil;
   if (args->doc != NIL)
     xmlFreeDoc(args->doc);
   return Qnil;
 }
-static VALUE parse_continue(ParseArgs *args);
+static VALUE parse_continue(VALUE parse_args);
 // Parse a string using gumbo_parse into a Nokogiri document
-static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE max_depth) {
+static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_attributes, VALUE max_errors, VALUE max_depth) {
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   options.max_tree_depth = NUM2INT(max_depth);
@@ -502,10 +527,13 @@ static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE m
     .url_or_frag = url,
     .doc = NIL,
   };
-  return rb_ensure(parse_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  return rb_ensure(parse_continue, parse_args, parse_cleanup, parse_args);
 }
-static VALUE parse_continue(ParseArgs *args) {
+static VALUE parse_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   xmlDocPtr doc;
   if (output->document->v.document.has_doctype) {
@@ -563,13 +591,14 @@ static xmlNodePtr extract_xml_node(VALUE node) {
 #endif
 }
-static VALUE fragment_continue(ParseArgs *args);
+static VALUE fragment_continue(VALUE parse_args);
 static VALUE fragment (
   VALUE self,
   VALUE doc_fragment,
   VALUE tags,
   VALUE ctx,
+  VALUE max_attributes,
   VALUE max_errors,
   VALUE max_depth
 ) {
@@ -676,6 +705,7 @@ static VALUE fragment (
   // Perform a fragment parse.
   int depth = NUM2INT(max_depth);
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   // Add one to account for the HTML element.
   options.max_tree_depth = depth < 0 ? -1 : (depth + 1);
@@ -692,11 +722,13 @@ static VALUE fragment (
     .url_or_frag = doc_fragment,
     .doc = (xmlDocPtr)extract_xml_node(doc),
   };
-  rb_ensure(fragment_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  rb_ensure(fragment_continue, parse_args, parse_cleanup, parse_args);
   return Qnil;
 }
-static VALUE fragment_continue(ParseArgs *args) {
+static VALUE fragment_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   VALUE doc_fragment = args->url_or_frag;
   xmlDocPtr xml_doc = args->doc;
@@ -720,10 +752,15 @@ void Init_nokogumbo() {
   VALUE mNokogiri = rb_const_get(rb_cObject, rb_intern_const("Nokogiri"));
   VALUE mNokogiriXml = rb_const_get(mNokogiri, rb_intern_const("XML"));
   cNokogiriXmlSyntaxError = rb_const_get(mNokogiriXml, rb_intern_const("SyntaxError"));
+  rb_gc_register_mark_object(cNokogiriXmlSyntaxError);
   cNokogiriXmlElement = rb_const_get(mNokogiriXml, rb_intern_const("Element"));
+  rb_gc_register_mark_object(cNokogiriXmlElement);
   cNokogiriXmlText = rb_const_get(mNokogiriXml, rb_intern_const("Text"));
+  rb_gc_register_mark_object(cNokogiriXmlText);
   cNokogiriXmlCData = rb_const_get(mNokogiriXml, rb_intern_const("CDATA"));
+  rb_gc_register_mark_object(cNokogiriXmlCData);
   cNokogiriXmlComment = rb_const_get(mNokogiriXml, rb_intern_const("Comment"));
+  rb_gc_register_mark_object(cNokogiriXmlComment);
   // Interned symbols.
   new = rb_intern_const("new");
@@ -736,6 +773,7 @@ void Init_nokogumbo() {
   // Class constants.
   VALUE HTML5 = rb_const_get(mNokogiri, rb_intern_const("HTML5"));
   Document = rb_const_get(HTML5, rb_intern_const("Document"));
+  rb_gc_register_mark_object(Document);
   // Interned symbols.
   internal_subset = rb_intern_const("internal_subset");
@@ -743,8 +781,8 @@ void Init_nokogumbo() {
   // Define Nokogumbo module with parse and fragment methods.
   VALUE Gumbo = rb_define_module("Nokogumbo");
-  rb_define_singleton_method(Gumbo, "parse", parse, 4);
-  rb_define_singleton_method(Gumbo, "fragment", fragment, 5);
+  rb_define_singleton_method(Gumbo, "parse", parse, 5);
+  rb_define_singleton_method(Gumbo, "fragment", fragment, 6);
   // Add private constant for testing.
   rb_define_const(Gumbo, "LINE_SUPPORTED", line_supported);

data/gumbo-parser/src/gumbo.h CHANGED

@@ -706,6 +706,15 @@ typedef struct GumboInternalOptions {
    */
   bool stop_on_first_error;
+  /**
+   * Maximum allowed number of attributes per element. If this limit is
+   * exceeded, the parser will return early with a partial document and
+   * the returned `GumboOutput` will have its `status` field set to
+   * `GUMBO_STATUS_TOO_MANY_ATTRIBUTES`. Set to `-1` to disable the limit.
+   * Default: `400`.
+   */
+  int max_attributes;
   /**
    * Maximum allowed depth for the parse tree. If this limit is exceeded,
    * the parser will return early with a partial document and the returned
@@ -796,6 +805,16 @@ typedef enum {
    */
   GUMBO_STATUS_TREE_TOO_DEEP,
+  /**
+   * Indicates that the maximum number of attributes per element
+   * (`GumboOptions::max_attributes`) was reached during parsing. The
+   * resulting tree will be a partial document, with no further nodes
+   * created after the point where the limit was reached. The partial
+   * document may be useful for constructing an error message but
+   * typically shouldn't be used for other purposes.
+   */
+  GUMBO_STATUS_TOO_MANY_ATTRIBUTES,
   // Currently unused
   GUMBO_STATUS_OUT_OF_MEMORY,
 } GumboOutputStatus;

data/gumbo-parser/src/parser.c CHANGED

@@ -48,6 +48,7 @@ typedef uint8_t TagSet[GUMBO_TAG_LAST + 1];
 const GumboOptions kGumboDefaultOptions = {
   .tab_stop = 8,
   .stop_on_first_error = false,
+  .max_attributes = 400,
   .max_tree_depth = 400,
   .max_errors = -1,
   .fragment_context = NULL,
@@ -4858,6 +4859,8 @@ const char* gumbo_status_to_string(GumboOutputStatus status) {
       return "OK";
     case GUMBO_STATUS_OUT_OF_MEMORY:
       return "System allocator returned NULL during parsing";
+    case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
+      return "Attributes per element limit exceeded";
     case GUMBO_STATUS_TREE_TOO_DEEP:
       return "Document tree depth limit exceeded";
     default:

data/gumbo-parser/src/tokenizer.c CHANGED

@@ -784,12 +784,22 @@ static void add_duplicate_attr_error(GumboParser* parser) {
 static void finish_attribute_name(GumboParser* parser) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   GumboTagState* tag_state = &tokenizer->_tag_state;
+  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
+  int max_attributes = parser->_options->max_attributes;
+  if (unlikely(max_attributes >= 0 && attributes->length >= (unsigned int) max_attributes)) {
+    parser->_output->status = GUMBO_STATUS_TOO_MANY_ATTRIBUTES;
+    gumbo_debug("Attributes limit exceeded.\n");
+    reinitialize_tag_buffer(parser);
+    tag_state->_drop_next_attr_value = true;
+    return;
+  }
   // May've been set by a previous attribute without a value; reset it here.
   tag_state->_drop_next_attr_value = false;
   assert(tag_state->_attributes.data);
   assert(tag_state->_attributes.capacity);
-  GumboVector* /* GumboAttribute* */ attributes = &tag_state->_attributes;
   for (unsigned int i = 0; i < attributes->length; ++i) {
     GumboAttribute* attr = attributes->data[i];
     if (

data/gumbo-parser/src/utf8.c CHANGED

@@ -193,6 +193,11 @@ void utf8iterator_init (
   iter->_pos.offset = 0;
   iter->_parser = parser;
   read_char(iter);
+  if (iter->_current == kUtf8BomChar) {
+    iter->_start += iter->_width;
+    iter->_pos.offset += iter->_width;
+    read_char(iter);
+  }
 }
 void utf8iterator_next(Utf8Iterator* iter) {

data/gumbo-parser/src/utf8.h CHANGED

@@ -31,6 +31,7 @@ struct GumboInternalParser;
 // Unicode replacement char.
 #define kUtf8ReplacementChar 0xFFFD
+#define kUtf8BomChar 0xFEFF
 #define kUtf8MaxChar 0x10FFFF
 typedef struct GumboInternalUtf8Iterator {

data/lib/nokogumbo.rb CHANGED

@@ -5,6 +5,9 @@ require 'nokogumbo/html5'
 require 'nokogumbo/nokogumbo'
 module Nokogumbo
+  # The default maximum number of attributes per element.
+  DEFAULT_MAX_ATTRIBUTES = 400
   # The default maximum number of errors for parsing a document or a fragment.
   DEFAULT_MAX_ERRORS = 0

data/lib/nokogumbo/html5.rb CHANGED

@@ -19,7 +19,7 @@ module Nokogiri
     # Parse an HTML 5 document. Convenience method for Nokogiri::HTML5::Document.parse
     def self.parse(string, url = nil, encoding = nil, **options, &block)
-      Document.parse(string, url, encoding, options, &block)
+      Document.parse(string, url, encoding, **options, &block)
     end
     # Parse a fragment from +string+. Convenience method for
@@ -92,19 +92,20 @@ module Nokogiri
         if encoding.nil?
           string = string.read
         else
-        string = string.read(encoding: encoding)
+          string = string.read(encoding: encoding)
         end
       else
         # Otherwise the string has the given encoding.
-        if encoding && string.respond_to?(:force_encoding)
+        string = string.to_str
+        if encoding
           string = string.dup
           string.force_encoding(encoding)
         end
       end
-      # convert to UTF-8 (Ruby 1.9+)
-      if string.respond_to?(:encoding) && string.encoding != Encoding::UTF_8
-        string = reencode(string.dup)
+      # convert to UTF-8
+      if string.encoding != Encoding::UTF_8
+        string = reencode(string)
       end
       string
     end
@@ -123,18 +124,17 @@ module Nokogiri
     # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
     #
     def self.reencode(body, content_type=nil)
-      return body unless body.respond_to? :encoding
       if body.encoding == Encoding::ASCII_8BIT
         encoding = nil
         # look for a Byte Order Mark (BOM)
-        if body[0..1] == "\xFE\xFF"
-          encoding = 'utf-16be'
-        elsif body[0..1] == "\xFF\xFE"
-          encoding = 'utf-16le'
-        elsif body[0..2] == "\xEF\xBB\xBF"
-          encoding = 'utf-8'
+        initial_bytes = body[0..2].bytes
+        if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
+          encoding = Encoding::UTF_8
+        elsif initial_bytes[0..1] == [0xFE, 0xFF]
+          encoding = Encoding::UTF_16BE
+        elsif initial_bytes[0..1] == [0xFF, 0xFE]
+          encoding = Encoding::UTF_16LE
         end
         # look for a charset in a content-encoding header
@@ -154,6 +154,7 @@ module Nokogiri
         encoding ||= Encoding::ISO_8859_1
         # change the encoding to match the detected or inferred encoding
+        body = body.dup
         begin
           body.force_encoding(encoding)
         rescue ArgumentError

data/lib/nokogumbo/html5/document.rb CHANGED

@@ -12,6 +12,9 @@ module Nokogiri
         if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
           url ||= string_or_io.path
         end
+        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+          raise ArgumentError.new("not a string or IO object")
+        end
         do_parse(string_or_io, url, encoding, options)
       end
@@ -21,7 +24,8 @@ module Nokogiri
       end
       def self.read_memory(string, url = nil, encoding = nil, **options)
-        do_parse(string.to_s, url, encoding, options)
+        raise ArgumentError.new("string object doesn't respond to :to_str") unless string.respond_to?(:to_str)
+        do_parse(string, url, encoding, options)
       end
       def fragment(tags = nil)
@@ -37,9 +41,10 @@ module Nokogiri
       private
       def self.do_parse(string_or_io, url, encoding, options)
         string = HTML5.read_and_encode(string_or_io, encoding)
+        max_attributes = options[:max_attributes] || Nokogumbo::DEFAULT_MAX_ATTRIBUTES
         max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogumbo::DEFAULT_MAX_ERRORS
         max_depth = options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogumbo.parse(string.to_s, url, max_errors, max_depth)
+        doc = Nokogumbo.parse(string, url, max_attributes, max_errors, max_depth)
         doc.encoding = 'UTF-8'
         doc
       end

data/lib/nokogumbo/html5/document_fragment.rb CHANGED

@@ -12,10 +12,11 @@ module Nokogiri
         self.errors = []
         return self unless tags
+        max_attributes = options[:max_attributes] || Nokogumbo::DEFAULT_MAX_ATTRIBUTES
         max_errors = options[:max_errors] || Nokogumbo::DEFAULT_MAX_ERRORS
         max_depth = options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH
         tags = Nokogiri::HTML5.read_and_encode(tags, nil)
-        Nokogumbo.fragment(self, tags, ctx, max_errors, max_depth)
+        Nokogumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
       def serialize(options = {}, &block)

data/lib/nokogumbo/version.rb CHANGED

@@ -1,3 +1,3 @@
 module Nokogumbo
-  VERSION = "2.0.2"
+  VERSION = "2.0.3"
 end

metadata CHANGED

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: nokogumbo
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Sam Ruby
 - Stephen Checkoway
-autorequire:
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2020-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -94,7 +94,7 @@ metadata:
   changelog_uri: https://github.com/rubys/nokogumbo/blob/master/CHANGELOG.md
   homepage_uri: https://github.com/rubys/nokogumbo/#readme
   source_code_uri: https://github.com/rubys/nokogumbo
-post_install_message:
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,8 +109,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key:
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Nokogiri interface to the Gumbo HTML5 parser
 test_files: []