RubyGems - nokogumbo - Versions diffs - 2.0.0 → 2.0.5 - Mend

nokogumbo 2.0.0 → 2.0.5

Files changed (17) hide show

checksums.yaml +4 -4
data/README.md +20 -4
data/ext/nokogumbo/extconf.rb +50 -27
data/ext/nokogumbo/nokogumbo.c +63 -14
data/gumbo-parser/src/error.c +17 -8
data/gumbo-parser/src/gumbo.h +27 -0
data/gumbo-parser/src/parser.c +476 -480
data/gumbo-parser/src/tokenizer.c +24 -27
data/gumbo-parser/src/tokenizer.h +2 -13
data/gumbo-parser/src/utf8.c +5 -0
data/gumbo-parser/src/utf8.h +1 -0
data/lib/nokogumbo.rb +22 -9
data/lib/nokogumbo/html5.rb +15 -14
data/lib/nokogumbo/html5/document.rb +7 -2
data/lib/nokogumbo/html5/document_fragment.rb +2 -1
data/lib/nokogumbo/version.rb +1 -1
metadata +3 -4

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97aae1603382eb4357f4126f4c36f86841b930a04b46e0eb6a1c02c26c9e77a4
-  data.tar.gz: 614f1dc01be03d4ccb48b43d512faf80f556e8ddde4690fa34fbb9e420c36cb9
+  metadata.gz: a84b367d94046358f7844781b0f92cea51a75e052d54e35b53ab03602743f1b8
+  data.tar.gz: 8d96a5adfa701f658f7ba193ee96bb8a7e6901c1ff4d3fb2dad6f3e372ce66d2
 SHA512:
-  metadata.gz: aabd005ec985f1a94b0b82195ce8547fabc3d3f97e672b9b03c174ddd131b534b6c2648803ccd0d1fdfc9224bab132cdbd2757271b5646fc3873649d1f509e26
-  data.tar.gz: 4e793d5436de772587f2abcdb54c1060a3ddbde3dfcad05539e823dbde3c67f0792d1589289323fe76d5baebe265a526aba71d5adcbeb8d2ca72559d789e7b14
+  metadata.gz: de2472c6ff89e3f0076a44ac13fa67688e82f909b265a2b70fe45225daf01aaf6059c6ca94f06e10ff94e10ac8a8f42b685e63f494849f04f3af56f337a73382
+  data.tar.gz: 3880defdaa15cb278236cf170d5727d1d73b14698f1ea41e7a7141da7a2fe8c3bafea19367196214c0dc0c1c27854602714d80abd30ecfd6be90f4277f3e33d7

data/README.md CHANGED Viewed

@@ -1,7 +1,7 @@
 # Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
-Nokogumbo provides the ability for a Ruby program to invoke the
-[Gumbo HTML5 parser](https://github.com/google/gumbo-parser#readme)
+Nokogumbo provides the ability for a Ruby program to invoke
+[our version of the  Gumbo HTML5 parser](https://github.com/rubys/nokogumbo/tree/master/gumbo-parser/src)
 and to access the result as a
 [Nokogiri::HTML::Document](http://rdoc.info/github/sparklemotion/nokogiri/Nokogiri/HTML/Document).
@@ -39,8 +39,8 @@ The document and fragment parsing methods,
 - `Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})`
 support options that are different from Nokogiri's.
-The two currently supported options are `:max_errors` and `:max_tree_depth`,
-described below.
+The three currently supported options are `:max_errors`, `:max_tree_depth` and
+`:max_attributes`, described below.
 ### Error reporting
 Nokogumbo contains an experimental parse error reporting facility. By default,
@@ -128,6 +128,22 @@ doc = Nokogiri.HTML5(html)
 doc = Nokogiri.HTML5(html, max_tree_depth: -1)
 ```
+### Attribute limit per element
+The maximum number of attributes per DOM element is configurable by the
+`:max_attributes` option. If a given element would exceed this limit, then an
+[ArgumentError](https://ruby-doc.org/core-2.5.0/ArgumentError.html) is thrown.
+This limit (which defaults to `Nokogumbo::DEFAULT_MAX_ATTRIBUTES = 400`) can
+be removed by giving the option `max_attributes: -1`.
+``` ruby
+html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
+# "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
+doc = Nokogiri.HTML5(html)
+# raises ArgumentError: Attributes per element limit exceeded
+doc = Nokogiri.HTML5(html, max_attributes: -1)
+```
 ## HTML Serialization
 After parsing HTML, it may be serialized using any of the Nokogiri

data/ext/nokogumbo/extconf.rb CHANGED Viewed

@@ -1,9 +1,11 @@
+require 'rubygems'
 require 'fileutils'
 require 'mkmf'
 require 'nokogiri'
 $CFLAGS += " -std=c99"
 $LDFLAGS.gsub!('-Wl,--no-undefined', '')
+$DLDFLAGS.gsub!('-Wl,--no-undefined', '')
 $warnflags = CONFIG['warnflags'] = '-Wall'
 NG_SPEC = Gem::Specification.find_by_name('nokogiri', "= #{Nokogiri::VERSION}")
@@ -23,7 +25,6 @@ def download_headers
     return nil if dep_index.nil?
     requirement = NG_SPEC.dependencies[dep_index].requirement.to_s
-    require 'rubygems'
     gem 'mini_portile2', requirement
     require 'mini_portile2'
     p = MiniPortile::new('libxml2', version).tap do |r|
@@ -60,41 +61,64 @@ end
 have_libxml2 = false
 have_ng = false
+def windows?
+  ::RUBY_PLATFORM =~ /mingw|mswin/
+end
+def modern_nokogiri?
+  nokogiri_version = Gem::Version.new(Nokogiri::VERSION)
+  requirement = windows? ? ">= 1.11.2" : ">= 1.11.0.rc4"
+  Gem::Requirement.new(requirement).satisfied_by?(nokogiri_version)
+end
 if !prohibited
-  if Nokogiri::VERSION_INFO.include?('libxml') and
-     Nokogiri::VERSION_INFO['libxml']['source'] == 'packaged'
-    # Nokogiri has libxml2 built in. Find the headers.
-    libxml2_path = File.join(Nokogiri::VERSION_INFO['libxml']['libxml2_path'],
-                             'include/libxml2')
-    if find_header('libxml/tree.h', libxml2_path)
-      have_libxml2 = true
+  if modern_nokogiri?
+    append_cflags(Nokogiri::VERSION_INFO["nokogiri"]["cppflags"])
+    append_ldflags(Nokogiri::VERSION_INFO["nokogiri"]["ldflags"]) # may be nil for nokogiri pre-1.11.2
+    have_libxml2 = if Nokogiri::VERSION_INFO["nokogiri"]["ldflags"].empty?
+                     have_header('libxml/tree.h')
+                   else
+                     have_func("xmlNewDoc", "libxml/tree.h")
+                   end
+  end
+  if !have_libxml2
+    if Nokogiri::VERSION_INFO.include?('libxml') and
+       Nokogiri::VERSION_INFO['libxml']['source'] == 'packaged'
+      # Nokogiri has libxml2 built in. Find the headers.
+      libxml2_path = File.join(Nokogiri::VERSION_INFO['libxml']['libxml2_path'],
+                               'include/libxml2')
+      if find_header('libxml/tree.h', libxml2_path)
+        have_libxml2 = true
+      else
+        # Unfortunately, some versions of Nokogiri delete these files.
+        # https://github.com/sparklemotion/nokogiri/pull/1788
+        # Try to download them
+        libxml2_path = download_headers
+        unless libxml2_path.nil?
+          have_libxml2 = find_header('libxml/tree.h', libxml2_path)
+        end
+      end
     else
-      # Unfortunately, some versions of Nokogiri delete these files.
-      # https://github.com/sparklemotion/nokogiri/pull/1788
-      # Try to download them
-      libxml2_path = download_headers
-      unless libxml2_path.nil?
-        have_libxml2 = find_header('libxml/tree.h', libxml2_path)
+      # Nokogiri is compiled with system headers.
+      # Hack to work around broken mkmf on macOS
+      # (https://bugs.ruby-lang.org/issues/14992 fixed now)
+      if RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] == 'DYLD_LIBRARY_PATH'
+        RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] = 'DYLD_FALLBACK_LIBRARY_PATH'
       end
-    end
-  else
-    # Nokogiri is compiled with system headers.
-    # Hack to work around broken mkmf on macOS
-    # (https://bugs.ruby-lang.org/issues/14992 fixed now)
-    if RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] == 'DYLD_LIBRARY_PATH'
-      RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] = 'DYLD_FALLBACK_LIBRARY_PATH'
-    end
-    pkg_config('libxml-2.0')
-    have_libxml2 = have_library('xml2', 'xmlNewDoc')
+      pkg_config('libxml-2.0')
+      have_libxml2 = have_library('xml2', 'xmlNewDoc')
+    end
   end
   if required and !have_libxml2
     abort "libxml2 required but could not be located"
   end
   if have_libxml2
-    # Find nokogiri.h
-    have_ng = find_header('nokogiri.h', File.join(NG_SPEC.gem_dir, 'ext/nokogiri'))
+    have_ng = have_header('nokogiri.h') || find_header('nokogiri.h', File.join(NG_SPEC.gem_dir, 'ext/nokogiri'))
   end
 end
@@ -104,7 +128,6 @@ end
 # Symlink gumbo-parser source files.
 ext_dir = File.dirname(__FILE__)
-gumbo_src = File.join(ext_dir, 'gumbo_src')
 Dir.chdir(ext_dir) do
   $srcs = Dir['*.c', '../../gumbo-parser/src/*.c']

data/ext/nokogumbo/nokogumbo.c CHANGED Viewed

@@ -33,6 +33,7 @@ static ID parent;
 /* Backwards compatibility to Ruby 2.1.0 */
 #if RUBY_API_VERSION_CODE < 20200
+#define ONIG_ESCAPE_UCHAR_COLLISION 1
 #include <ruby/encoding.h>
 static VALUE rb_utf8_str_new(const char *str, long length) {
@@ -280,6 +281,7 @@ static GumboOutput *perform_parse(const GumboOptions *options, VALUE input) {
   switch (output->status) {
   case GUMBO_STATUS_OK:
     break;
+  case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
   case GUMBO_STATUS_TREE_TOO_DEEP:
     gumbo_destroy_output(output);
     rb_raise(rb_eArgError, "%s", status_string);
@@ -385,7 +387,7 @@ static void build_tree (
       case GUMBO_NODE_ELEMENT:
       {
         xml_child = xmlNewDocNode(doc, NIL, BAD_CAST gumbo_child->v.element.name, NULL);
-        set_line(xml_child, gumbo_child->v.text.start_pos.line);
+        set_line(xml_child, gumbo_child->v.element.start_pos.line);
         if (xml_root == NIL)
           xml_root = xml_child;
         xmlNsPtr ns = NIL;
@@ -478,19 +480,43 @@ typedef struct {
   xmlDocPtr doc;
 } ParseArgs;
-static VALUE parse_cleanup(ParseArgs *args) {
+static void parse_args_mark(void *parse_args) {
+  ParseArgs *args = parse_args;
+  rb_gc_mark_maybe(args->input);
+  rb_gc_mark_maybe(args->url_or_frag);
+}
+// Wrap a ParseArgs pointer. The underlying ParseArgs must outlive the
+// wrapper.
+static VALUE wrap_parse_args(ParseArgs *args) {
+  return Data_Wrap_Struct(rb_cData, parse_args_mark, RUBY_NEVER_FREE, args);
+}
+// Returnsd the underlying ParseArgs wrapped by wrap_parse_args.
+static ParseArgs *unwrap_parse_args(VALUE obj) {
+  ParseArgs *args;
+  Data_Get_Struct(obj, ParseArgs, args);
+  return args;
+}
+static VALUE parse_cleanup(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   gumbo_destroy_output(args->output);
+  // Make sure garbage collection doesn't mark the objects as being live based
+  // on references from the ParseArgs. This may be unnecessary.
+  args->input = Qnil;
+  args->url_or_frag = Qnil;
   if (args->doc != NIL)
     xmlFreeDoc(args->doc);
   return Qnil;
 }
-static VALUE parse_continue(ParseArgs *args);
+static VALUE parse_continue(VALUE parse_args);
 // Parse a string using gumbo_parse into a Nokogiri document
-static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE max_depth) {
+static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_attributes, VALUE max_errors, VALUE max_depth) {
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   options.max_tree_depth = NUM2INT(max_depth);
@@ -501,10 +527,13 @@ static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE m
     .url_or_frag = url,
     .doc = NIL,
   };
-  return rb_ensure(parse_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  return rb_ensure(parse_continue, parse_args, parse_cleanup, parse_args);
 }
-static VALUE parse_continue(ParseArgs *args) {
+static VALUE parse_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   xmlDocPtr doc;
   if (output->document->v.document.has_doctype) {
@@ -562,13 +591,14 @@ static xmlNodePtr extract_xml_node(VALUE node) {
 #endif
 }
-static VALUE fragment_continue(ParseArgs *args);
+static VALUE fragment_continue(VALUE parse_args);
 static VALUE fragment (
   VALUE self,
   VALUE doc_fragment,
   VALUE tags,
   VALUE ctx,
+  VALUE max_attributes,
   VALUE max_errors,
   VALUE max_depth
 ) {
@@ -675,6 +705,7 @@ static VALUE fragment (
   // Perform a fragment parse.
   int depth = NUM2INT(max_depth);
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   // Add one to account for the HTML element.
   options.max_tree_depth = depth < 0 ? -1 : (depth + 1);
@@ -691,11 +722,13 @@ static VALUE fragment (
     .url_or_frag = doc_fragment,
     .doc = (xmlDocPtr)extract_xml_node(doc),
   };
-  rb_ensure(fragment_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  rb_ensure(fragment_continue, parse_args, parse_cleanup, parse_args);
   return Qnil;
 }
-static VALUE fragment_continue(ParseArgs *args) {
+static VALUE fragment_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   VALUE doc_fragment = args->url_or_frag;
   xmlDocPtr xml_doc = args->doc;
@@ -709,27 +742,38 @@ static VALUE fragment_continue(ParseArgs *args) {
 // Initialize the Nokogumbo class and fetch constants we will use later.
 void Init_nokogumbo() {
-  rb_funcall(rb_mKernel, rb_intern("gem"), 1, rb_utf8_str_new_static("nokogiri", 8));
+  rb_funcall(rb_mKernel, rb_intern_const("gem"), 1, rb_utf8_str_new_static("nokogiri", 8));
   rb_require("nokogiri");
-#ifndef NGLIB
+  VALUE line_supported = Qtrue;
+#if !NGLIB
   // Class constants.
   VALUE mNokogiri = rb_const_get(rb_cObject, rb_intern_const("Nokogiri"));
   VALUE mNokogiriXml = rb_const_get(mNokogiri, rb_intern_const("XML"));
   cNokogiriXmlSyntaxError = rb_const_get(mNokogiriXml, rb_intern_const("SyntaxError"));
+  rb_gc_register_mark_object(cNokogiriXmlSyntaxError);
   cNokogiriXmlElement = rb_const_get(mNokogiriXml, rb_intern_const("Element"));
+  rb_gc_register_mark_object(cNokogiriXmlElement);
   cNokogiriXmlText = rb_const_get(mNokogiriXml, rb_intern_const("Text"));
+  rb_gc_register_mark_object(cNokogiriXmlText);
   cNokogiriXmlCData = rb_const_get(mNokogiriXml, rb_intern_const("CDATA"));
+  rb_gc_register_mark_object(cNokogiriXmlCData);
   cNokogiriXmlComment = rb_const_get(mNokogiriXml, rb_intern_const("Comment"));
+  rb_gc_register_mark_object(cNokogiriXmlComment);
   // Interned symbols.
   new = rb_intern_const("new");
   node_name_ = rb_intern_const("node_name=");
+  // #line is not supported (returns 0)
+  line_supported = Qfalse;
 #endif
   // Class constants.
   VALUE HTML5 = rb_const_get(mNokogiri, rb_intern_const("HTML5"));
   Document = rb_const_get(HTML5, rb_intern_const("Document"));
+  rb_gc_register_mark_object(Document);
   // Interned symbols.
   internal_subset = rb_intern_const("internal_subset");
@@ -737,8 +781,13 @@ void Init_nokogumbo() {
   // Define Nokogumbo module with parse and fragment methods.
   VALUE Gumbo = rb_define_module("Nokogumbo");
-  rb_define_singleton_method(Gumbo, "parse", parse, 4);
-  rb_define_singleton_method(Gumbo, "fragment", fragment, 5);
+  rb_define_singleton_method(Gumbo, "parse", parse, 5);
+  rb_define_singleton_method(Gumbo, "fragment", fragment, 6);
+  // Add private constant for testing.
+  rb_define_const(Gumbo, "LINE_SUPPORTED", line_supported);
+  rb_funcall(Gumbo, rb_intern_const("private_constant"), 1,
+             rb_utf8_str_new_cstr("LINE_SUPPORTED"));
 }
 // vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab:

data/gumbo-parser/src/error.c CHANGED Viewed

@@ -365,11 +365,14 @@ static void handle_parser_error (
 // pointer to the beginning of the string if this is the first line.
 static const char* find_prev_newline (
   const char* source_text,
+  size_t source_length,
   const char* error_location
 ) {
+  const char* source_end = source_text + source_length;
   assert(error_location >= source_text);
+  assert(error_location <= source_end);
   const char* c = error_location;
-  if (*c == '\n' && c != source_text)
+  if (c != source_text && (error_location == source_end || *c == '\n'))
     --c;
   while (c != source_text && *c != '\n')
     --c;
@@ -377,20 +380,25 @@ static const char* find_prev_newline (
 }
 // Finds the next newline in the original source buffer from a given byte
-// location. Returns a character pointer to that newline, or a pointer to the
-// terminating null byte if this is the last line.
+// location. Returns a character pointer to that newline, or a pointer to
+// source_text + source_length if this is the last line.
 static const char* find_next_newline(
-  const char* source_text_end,
+  const char* source_text,
+  size_t source_length,
   const char* error_location
 ) {
-  assert(error_location <= source_text_end);
+  const char* source_end = source_text + source_length;
+  assert(error_location >= source_text);
+  assert(error_location <= source_end);
   const char* c = error_location;
-  while (c != source_text_end && *c != '\n')
+  while (c != source_end && *c != '\n')
     ++c;
   return c;
 }
 GumboError* gumbo_add_error(GumboParser* parser) {
+  parser->_output->document_error = true;
   int max_errors = parser->_options->max_errors;
   if (max_errors >= 0 && parser->_output->errors.length >= (unsigned int) max_errors) {
     return NULL;
@@ -547,8 +555,9 @@ void caret_diagnostic_to_string (
 ) {
   error_to_string(error, output);
-  const char* line_start = find_prev_newline(source_text, error->original_text.data);
-  const char* line_end = find_next_newline(source_text + source_length, error->original_text.data);
+  const char* error_text = error->original_text.data;
+  const char* line_start = find_prev_newline(source_text, source_length, error_text);
+  const char* line_end = find_next_newline(source_text, source_length, error_text);
   GumboStringPiece original_line;
   original_line.data = line_start;
   original_line.length = line_end - line_start;

data/gumbo-parser/src/gumbo.h CHANGED Viewed

@@ -706,6 +706,15 @@ typedef struct GumboInternalOptions {
    */
   bool stop_on_first_error;
+  /**
+   * Maximum allowed number of attributes per element. If this limit is
+   * exceeded, the parser will return early with a partial document and
+   * the returned `GumboOutput` will have its `status` field set to
+   * `GUMBO_STATUS_TOO_MANY_ATTRIBUTES`. Set to `-1` to disable the limit.
+   * Default: `400`.
+   */
+  int max_attributes;
   /**
    * Maximum allowed depth for the parse tree. If this limit is exceeded,
    * the parser will return early with a partial document and the returned
@@ -796,6 +805,16 @@ typedef enum {
    */
   GUMBO_STATUS_TREE_TOO_DEEP,
+  /**
+   * Indicates that the maximum number of attributes per element
+   * (`GumboOptions::max_attributes`) was reached during parsing. The
+   * resulting tree will be a partial document, with no further nodes
+   * created after the point where the limit was reached. The partial
+   * document may be useful for constructing an error message but
+   * typically shouldn't be used for other purposes.
+   */
+  GUMBO_STATUS_TOO_MANY_ATTRIBUTES,
   // Currently unused
   GUMBO_STATUS_OUT_OF_MEMORY,
 } GumboOutputStatus;
@@ -820,6 +839,14 @@ typedef struct GumboInternalOutput {
    */
   GumboVector /* GumboError */ errors;
+  /**
+   * True if the parser encounted an error.
+   *
+   * This can be true and `errors` an empty `GumboVector` if the `max_errors`
+   * option was set to 0.
+   */
+  bool document_error;
   /**
    * A status code indicating whether parsing finished successfully or was
    * stopped mid-document due to exceptional circumstances.