RubyGems - nokogumbo - Versions diffs - 2.0.0 → 2.0.5 - Mend

nokogumbo 2.0.0 → 2.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (17) hide show

checksums.yaml +4 -4
data/README.md +20 -4
data/ext/nokogumbo/extconf.rb +50 -27
data/ext/nokogumbo/nokogumbo.c +63 -14
data/gumbo-parser/src/error.c +17 -8
data/gumbo-parser/src/gumbo.h +27 -0
data/gumbo-parser/src/parser.c +476 -480
data/gumbo-parser/src/tokenizer.c +24 -27
data/gumbo-parser/src/tokenizer.h +2 -13
data/gumbo-parser/src/utf8.c +5 -0
data/gumbo-parser/src/utf8.h +1 -0
data/lib/nokogumbo.rb +22 -9
data/lib/nokogumbo/html5.rb +15 -14
data/lib/nokogumbo/html5/document.rb +7 -2
data/lib/nokogumbo/html5/document_fragment.rb +2 -1
data/lib/nokogumbo/version.rb +1 -1
metadata +3 -4

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97aae1603382eb4357f4126f4c36f86841b930a04b46e0eb6a1c02c26c9e77a4
-  data.tar.gz: 614f1dc01be03d4ccb48b43d512faf80f556e8ddde4690fa34fbb9e420c36cb9
+  metadata.gz: a84b367d94046358f7844781b0f92cea51a75e052d54e35b53ab03602743f1b8
+  data.tar.gz: 8d96a5adfa701f658f7ba193ee96bb8a7e6901c1ff4d3fb2dad6f3e372ce66d2
 SHA512:
-  metadata.gz: aabd005ec985f1a94b0b82195ce8547fabc3d3f97e672b9b03c174ddd131b534b6c2648803ccd0d1fdfc9224bab132cdbd2757271b5646fc3873649d1f509e26
-  data.tar.gz: 4e793d5436de772587f2abcdb54c1060a3ddbde3dfcad05539e823dbde3c67f0792d1589289323fe76d5baebe265a526aba71d5adcbeb8d2ca72559d789e7b14
+  metadata.gz: de2472c6ff89e3f0076a44ac13fa67688e82f909b265a2b70fe45225daf01aaf6059c6ca94f06e10ff94e10ac8a8f42b685e63f494849f04f3af56f337a73382
+  data.tar.gz: 3880defdaa15cb278236cf170d5727d1d73b14698f1ea41e7a7141da7a2fe8c3bafea19367196214c0dc0c1c27854602714d80abd30ecfd6be90f4277f3e33d7

data/README.md CHANGED Viewed

@@ -1,7 +1,7 @@
 # Nokogumbo - a Nokogiri interface to the Gumbo HTML5 parser.
-Nokogumbo provides the ability for a Ruby program to invoke the
-[Gumbo HTML5 parser](https://github.com/google/gumbo-parser#readme)
+Nokogumbo provides the ability for a Ruby program to invoke
+[our version of the  Gumbo HTML5 parser](https://github.com/rubys/nokogumbo/tree/master/gumbo-parser/src)
 and to access the result as a
 [Nokogiri::HTML::Document](http://rdoc.info/github/sparklemotion/nokogiri/Nokogiri/HTML/Document).
@@ -39,8 +39,8 @@ The document and fragment parsing methods,
 - `Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})`
 support options that are different from Nokogiri's.
-The two currently supported options are `:max_errors` and `:max_tree_depth`,
-described below.
+The three currently supported options are `:max_errors`, `:max_tree_depth` and
+`:max_attributes`, described below.
 ### Error reporting
 Nokogumbo contains an experimental parse error reporting facility. By default,
@@ -128,6 +128,22 @@ doc = Nokogiri.HTML5(html)
 doc = Nokogiri.HTML5(html, max_tree_depth: -1)
 ```
+### Attribute limit per element
+The maximum number of attributes per DOM element is configurable by the
+`:max_attributes` option. If a given element would exceed this limit, then an
+[ArgumentError](https://ruby-doc.org/core-2.5.0/ArgumentError.html) is thrown.
+This limit (which defaults to `Nokogumbo::DEFAULT_MAX_ATTRIBUTES = 400`) can
+be removed by giving the option `max_attributes: -1`.
+``` ruby
+html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>'
+# "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>"
+doc = Nokogiri.HTML5(html)
+# raises ArgumentError: Attributes per element limit exceeded
+doc = Nokogiri.HTML5(html, max_attributes: -1)
+```
 ## HTML Serialization
 After parsing HTML, it may be serialized using any of the Nokogiri

data/ext/nokogumbo/extconf.rb CHANGED Viewed

@@ -1,9 +1,11 @@
+require 'rubygems'
 require 'fileutils'
 require 'mkmf'
 require 'nokogiri'
 $CFLAGS += " -std=c99"
 $LDFLAGS.gsub!('-Wl,--no-undefined', '')
+$DLDFLAGS.gsub!('-Wl,--no-undefined', '')
 $warnflags = CONFIG['warnflags'] = '-Wall'
 NG_SPEC = Gem::Specification.find_by_name('nokogiri', "= #{Nokogiri::VERSION}")
@@ -23,7 +25,6 @@ def download_headers
     return nil if dep_index.nil?
     requirement = NG_SPEC.dependencies[dep_index].requirement.to_s
-    require 'rubygems'
     gem 'mini_portile2', requirement
     require 'mini_portile2'
     p = MiniPortile::new('libxml2', version).tap do |r|
@@ -60,41 +61,64 @@ end
 have_libxml2 = false
 have_ng = false
+def windows?
+  ::RUBY_PLATFORM =~ /mingw|mswin/
+end
+def modern_nokogiri?
+  nokogiri_version = Gem::Version.new(Nokogiri::VERSION)
+  requirement = windows? ? ">= 1.11.2" : ">= 1.11.0.rc4"
+  Gem::Requirement.new(requirement).satisfied_by?(nokogiri_version)
+end
 if !prohibited
-  if Nokogiri::VERSION_INFO.include?('libxml') and
-     Nokogiri::VERSION_INFO['libxml']['source'] == 'packaged'
-    # Nokogiri has libxml2 built in. Find the headers.
-    libxml2_path = File.join(Nokogiri::VERSION_INFO['libxml']['libxml2_path'],
-                             'include/libxml2')
-    if find_header('libxml/tree.h', libxml2_path)
-      have_libxml2 = true
+  if modern_nokogiri?
+    append_cflags(Nokogiri::VERSION_INFO["nokogiri"]["cppflags"])
+    append_ldflags(Nokogiri::VERSION_INFO["nokogiri"]["ldflags"]) # may be nil for nokogiri pre-1.11.2
+    have_libxml2 = if Nokogiri::VERSION_INFO["nokogiri"]["ldflags"].empty?
+                     have_header('libxml/tree.h')
+                   else
+                     have_func("xmlNewDoc", "libxml/tree.h")
+                   end
+  end
+  if !have_libxml2
+    if Nokogiri::VERSION_INFO.include?('libxml') and
+       Nokogiri::VERSION_INFO['libxml']['source'] == 'packaged'
+      # Nokogiri has libxml2 built in. Find the headers.
+      libxml2_path = File.join(Nokogiri::VERSION_INFO['libxml']['libxml2_path'],
+                               'include/libxml2')
+      if find_header('libxml/tree.h', libxml2_path)
+        have_libxml2 = true
+      else
+        # Unfortunately, some versions of Nokogiri delete these files.
+        # https://github.com/sparklemotion/nokogiri/pull/1788
+        # Try to download them
+        libxml2_path = download_headers
+        unless libxml2_path.nil?
+          have_libxml2 = find_header('libxml/tree.h', libxml2_path)
+        end
+      end
     else
-      # Unfortunately, some versions of Nokogiri delete these files.
-      # https://github.com/sparklemotion/nokogiri/pull/1788
-      # Try to download them
-      libxml2_path = download_headers
-      unless libxml2_path.nil?
-        have_libxml2 = find_header('libxml/tree.h', libxml2_path)
+      # Nokogiri is compiled with system headers.
+      # Hack to work around broken mkmf on macOS
+      # (https://bugs.ruby-lang.org/issues/14992 fixed now)
+      if RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] == 'DYLD_LIBRARY_PATH'
+        RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] = 'DYLD_FALLBACK_LIBRARY_PATH'
       end
-    end
-  else
-    # Nokogiri is compiled with system headers.
-    # Hack to work around broken mkmf on macOS
-    # (https://bugs.ruby-lang.org/issues/14992 fixed now)
-    if RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] == 'DYLD_LIBRARY_PATH'
-      RbConfig::MAKEFILE_CONFIG['LIBPATHENV'] = 'DYLD_FALLBACK_LIBRARY_PATH'
-    end
-    pkg_config('libxml-2.0')
-    have_libxml2 = have_library('xml2', 'xmlNewDoc')
+      pkg_config('libxml-2.0')
+      have_libxml2 = have_library('xml2', 'xmlNewDoc')
+    end
   end
   if required and !have_libxml2
     abort "libxml2 required but could not be located"
   end
   if have_libxml2
-    # Find nokogiri.h
-    have_ng = find_header('nokogiri.h', File.join(NG_SPEC.gem_dir, 'ext/nokogiri'))
+    have_ng = have_header('nokogiri.h') || find_header('nokogiri.h', File.join(NG_SPEC.gem_dir, 'ext/nokogiri'))
   end
 end
@@ -104,7 +128,6 @@ end
 # Symlink gumbo-parser source files.
 ext_dir = File.dirname(__FILE__)
-gumbo_src = File.join(ext_dir, 'gumbo_src')
 Dir.chdir(ext_dir) do
   $srcs = Dir['*.c', '../../gumbo-parser/src/*.c']

data/ext/nokogumbo/nokogumbo.c CHANGED Viewed

@@ -33,6 +33,7 @@ static ID parent;
 /* Backwards compatibility to Ruby 2.1.0 */
 #if RUBY_API_VERSION_CODE < 20200
+#define ONIG_ESCAPE_UCHAR_COLLISION 1
 #include <ruby/encoding.h>
 static VALUE rb_utf8_str_new(const char *str, long length) {
@@ -280,6 +281,7 @@ static GumboOutput *perform_parse(const GumboOptions *options, VALUE input) {
   switch (output->status) {
   case GUMBO_STATUS_OK:
     break;
+  case GUMBO_STATUS_TOO_MANY_ATTRIBUTES:
   case GUMBO_STATUS_TREE_TOO_DEEP:
     gumbo_destroy_output(output);
     rb_raise(rb_eArgError, "%s", status_string);
@@ -385,7 +387,7 @@ static void build_tree (
       case GUMBO_NODE_ELEMENT:
       {
         xml_child = xmlNewDocNode(doc, NIL, BAD_CAST gumbo_child->v.element.name, NULL);
-        set_line(xml_child, gumbo_child->v.text.start_pos.line);
+        set_line(xml_child, gumbo_child->v.element.start_pos.line);
         if (xml_root == NIL)
           xml_root = xml_child;
         xmlNsPtr ns = NIL;
@@ -478,19 +480,43 @@ typedef struct {
   xmlDocPtr doc;
 } ParseArgs;
-static VALUE parse_cleanup(ParseArgs *args) {
+static void parse_args_mark(void *parse_args) {
+  ParseArgs *args = parse_args;
+  rb_gc_mark_maybe(args->input);
+  rb_gc_mark_maybe(args->url_or_frag);
+}
+// Wrap a ParseArgs pointer. The underlying ParseArgs must outlive the
+// wrapper.
+static VALUE wrap_parse_args(ParseArgs *args) {
+  return Data_Wrap_Struct(rb_cData, parse_args_mark, RUBY_NEVER_FREE, args);
+}
+// Returnsd the underlying ParseArgs wrapped by wrap_parse_args.
+static ParseArgs *unwrap_parse_args(VALUE obj) {
+  ParseArgs *args;
+  Data_Get_Struct(obj, ParseArgs, args);
+  return args;
+}
+static VALUE parse_cleanup(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   gumbo_destroy_output(args->output);
+  // Make sure garbage collection doesn't mark the objects as being live based
+  // on references from the ParseArgs. This may be unnecessary.
+  args->input = Qnil;
+  args->url_or_frag = Qnil;
   if (args->doc != NIL)
     xmlFreeDoc(args->doc);
   return Qnil;
 }
-static VALUE parse_continue(ParseArgs *args);
+static VALUE parse_continue(VALUE parse_args);
 // Parse a string using gumbo_parse into a Nokogiri document
-static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE max_depth) {
+static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_attributes, VALUE max_errors, VALUE max_depth) {
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   options.max_tree_depth = NUM2INT(max_depth);
@@ -501,10 +527,13 @@ static VALUE parse(VALUE self, VALUE input, VALUE url, VALUE max_errors, VALUE m
     .url_or_frag = url,
     .doc = NIL,
   };
-  return rb_ensure(parse_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  return rb_ensure(parse_continue, parse_args, parse_cleanup, parse_args);
 }
-static VALUE parse_continue(ParseArgs *args) {
+static VALUE parse_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   xmlDocPtr doc;
   if (output->document->v.document.has_doctype) {
@@ -562,13 +591,14 @@ static xmlNodePtr extract_xml_node(VALUE node) {
 #endif
 }
-static VALUE fragment_continue(ParseArgs *args);
+static VALUE fragment_continue(VALUE parse_args);
 static VALUE fragment (
   VALUE self,
   VALUE doc_fragment,
   VALUE tags,
   VALUE ctx,
+  VALUE max_attributes,
   VALUE max_errors,
   VALUE max_depth
 ) {
@@ -675,6 +705,7 @@ static VALUE fragment (
   // Perform a fragment parse.
   int depth = NUM2INT(max_depth);
   GumboOptions options = kGumboDefaultOptions;
+  options.max_attributes = NUM2INT(max_attributes);
   options.max_errors = NUM2INT(max_errors);
   // Add one to account for the HTML element.
   options.max_tree_depth = depth < 0 ? -1 : (depth + 1);
@@ -691,11 +722,13 @@ static VALUE fragment (
     .url_or_frag = doc_fragment,
     .doc = (xmlDocPtr)extract_xml_node(doc),
   };
-  rb_ensure(fragment_continue, (VALUE)&args, parse_cleanup, (VALUE)&args);
+  VALUE parse_args = wrap_parse_args(&args);
+  rb_ensure(fragment_continue, parse_args, parse_cleanup, parse_args);
   return Qnil;
 }
-static VALUE fragment_continue(ParseArgs *args) {
+static VALUE fragment_continue(VALUE parse_args) {
+  ParseArgs *args = unwrap_parse_args(parse_args);
   GumboOutput *output = args->output;
   VALUE doc_fragment = args->url_or_frag;
   xmlDocPtr xml_doc = args->doc;
@@ -709,27 +742,38 @@ static VALUE fragment_continue(ParseArgs *args) {
 // Initialize the Nokogumbo class and fetch constants we will use later.
 void Init_nokogumbo() {
-  rb_funcall(rb_mKernel, rb_intern("gem"), 1, rb_utf8_str_new_static("nokogiri", 8));
+  rb_funcall(rb_mKernel, rb_intern_const("gem"), 1, rb_utf8_str_new_static("nokogiri", 8));
   rb_require("nokogiri");
-#ifndef NGLIB
+  VALUE line_supported = Qtrue;
+#if !NGLIB
   // Class constants.
   VALUE mNokogiri = rb_const_get(rb_cObject, rb_intern_const("Nokogiri"));
   VALUE mNokogiriXml = rb_const_get(mNokogiri, rb_intern_const("XML"));
   cNokogiriXmlSyntaxError = rb_const_get(mNokogiriXml, rb_intern_const("SyntaxError"));
+  rb_gc_register_mark_object(cNokogiriXmlSyntaxError);
   cNokogiriXmlElement = rb_const_get(mNokogiriXml, rb_intern_const("Element"));
+  rb_gc_register_mark_object(cNokogiriXmlElement);
   cNokogiriXmlText = rb_const_get(mNokogiriXml, rb_intern_const("Text"));
+  rb_gc_register_mark_object(cNokogiriXmlText);
   cNokogiriXmlCData = rb_const_get(mNokogiriXml, rb_intern_const("CDATA"));
+  rb_gc_register_mark_object(cNokogiriXmlCData);
   cNokogiriXmlComment = rb_const_get(mNokogiriXml, rb_intern_const("Comment"));
+  rb_gc_register_mark_object(cNokogiriXmlComment);
   // Interned symbols.
   new = rb_intern_const("new");
   node_name_ = rb_intern_const("node_name=");
+  // #line is not supported (returns 0)
+  line_supported = Qfalse;
 #endif
   // Class constants.
   VALUE HTML5 = rb_const_get(mNokogiri, rb_intern_const("HTML5"));
   Document = rb_const_get(HTML5, rb_intern_const("Document"));
+  rb_gc_register_mark_object(Document);
   // Interned symbols.
   internal_subset = rb_intern_const("internal_subset");
@@ -737,8 +781,13 @@ void Init_nokogumbo() {
   // Define Nokogumbo module with parse and fragment methods.
   VALUE Gumbo = rb_define_module("Nokogumbo");
-  rb_define_singleton_method(Gumbo, "parse", parse, 4);
-  rb_define_singleton_method(Gumbo, "fragment", fragment, 5);
+  rb_define_singleton_method(Gumbo, "parse", parse, 5);
+  rb_define_singleton_method(Gumbo, "fragment", fragment, 6);
+  // Add private constant for testing.
+  rb_define_const(Gumbo, "LINE_SUPPORTED", line_supported);
+  rb_funcall(Gumbo, rb_intern_const("private_constant"), 1,
+             rb_utf8_str_new_cstr("LINE_SUPPORTED"));
 }
 // vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab:

data/gumbo-parser/src/error.c CHANGED Viewed

@@ -365,11 +365,14 @@ static void handle_parser_error (
 // pointer to the beginning of the string if this is the first line.
 static const char* find_prev_newline (
   const char* source_text,
+  size_t source_length,
   const char* error_location
 ) {
+  const char* source_end = source_text + source_length;
   assert(error_location >= source_text);
+  assert(error_location <= source_end);
   const char* c = error_location;
-  if (*c == '\n' && c != source_text)
+  if (c != source_text && (error_location == source_end || *c == '\n'))
     --c;
   while (c != source_text && *c != '\n')
     --c;
@@ -377,20 +380,25 @@ static const char* find_prev_newline (
 }
 // Finds the next newline in the original source buffer from a given byte
-// location. Returns a character pointer to that newline, or a pointer to the
-// terminating null byte if this is the last line.
+// location. Returns a character pointer to that newline, or a pointer to
+// source_text + source_length if this is the last line.
 static const char* find_next_newline(
-  const char* source_text_end,
+  const char* source_text,
+  size_t source_length,
   const char* error_location
 ) {
-  assert(error_location <= source_text_end);
+  const char* source_end = source_text + source_length;
+  assert(error_location >= source_text);
+  assert(error_location <= source_end);
   const char* c = error_location;
-  while (c != source_text_end && *c != '\n')
+  while (c != source_end && *c != '\n')
     ++c;
   return c;
 }
 GumboError* gumbo_add_error(GumboParser* parser) {
+  parser->_output->document_error = true;
   int max_errors = parser->_options->max_errors;
   if (max_errors >= 0 && parser->_output->errors.length >= (unsigned int) max_errors) {
     return NULL;
@@ -547,8 +555,9 @@ void caret_diagnostic_to_string (
 ) {
   error_to_string(error, output);
-  const char* line_start = find_prev_newline(source_text, error->original_text.data);
-  const char* line_end = find_next_newline(source_text + source_length, error->original_text.data);
+  const char* error_text = error->original_text.data;
+  const char* line_start = find_prev_newline(source_text, source_length, error_text);
+  const char* line_end = find_next_newline(source_text, source_length, error_text);
   GumboStringPiece original_line;
   original_line.data = line_start;
   original_line.length = line_end - line_start;

data/gumbo-parser/src/gumbo.h CHANGED Viewed

@@ -706,6 +706,15 @@ typedef struct GumboInternalOptions {
    */
   bool stop_on_first_error;
+  /**
+   * Maximum allowed number of attributes per element. If this limit is
+   * exceeded, the parser will return early with a partial document and
+   * the returned `GumboOutput` will have its `status` field set to
+   * `GUMBO_STATUS_TOO_MANY_ATTRIBUTES`. Set to `-1` to disable the limit.
+   * Default: `400`.
+   */
+  int max_attributes;
   /**
    * Maximum allowed depth for the parse tree. If this limit is exceeded,
    * the parser will return early with a partial document and the returned
@@ -796,6 +805,16 @@ typedef enum {
    */
   GUMBO_STATUS_TREE_TOO_DEEP,
+  /**
+   * Indicates that the maximum number of attributes per element
+   * (`GumboOptions::max_attributes`) was reached during parsing. The
+   * resulting tree will be a partial document, with no further nodes
+   * created after the point where the limit was reached. The partial
+   * document may be useful for constructing an error message but
+   * typically shouldn't be used for other purposes.
+   */
+  GUMBO_STATUS_TOO_MANY_ATTRIBUTES,
   // Currently unused
   GUMBO_STATUS_OUT_OF_MEMORY,
 } GumboOutputStatus;
@@ -820,6 +839,14 @@ typedef struct GumboInternalOutput {
    */
   GumboVector /* GumboError */ errors;
+  /**
+   * True if the parser encounted an error.
+   *
+   * This can be true and `errors` an empty `GumboVector` if the `max_errors`
+   * option was set to 0.
+   */
+  bool document_error;
   /**
    * A status code indicating whether parsing finished successfully or was
    * stopped mid-document due to exceptional circumstances.