nokogumbo 1.3.0 → 1.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 96fa61565f78d5491e0b6d5b505cf936524745eb848b8b6584fc15e20c7ae35b
+  data.tar.gz: e5416f71bbe90323f04b8aad4dc48b28947e43a9eb46f446f8ca1444f519a07b
+SHA512:
+  metadata.gz: 676bf3585d38cd4ad5c72b8b3afd4952e248c747683ae1072dd43f6ce1ccd279177e4d0c75a9821ed76d32806333128152231349d8d113ae5d81279580b13004
+  data.tar.gz: 3459078d96977399e75551c4a3ee5623091f48569984b771e540ec111125f5af91e39a8d78cbd3ce9280326b1b9395dc4a0b0d7f0a72294876682cb9fe35e3d9

data/README.md

@@ -36,7 +36,13 @@ Example
 -----
 ```ruby
 require 'nokogumbo'
-puts Nokogiri::HTML5.get('http://nokogiri.org').at('h1 abbr')['title']
+puts Nokogiri::HTML5.get('http://nokogiri.org').search('ol li')[2].text
+```
+
+Use `.to_html` instead of `.to_s` when parsing and serializing multiple times
+```
+require 'nokogumbo'
+Nokogiri::HTML5.parse(Nokogiri::HTML5.parse('<div></div> a').to_html).to_html
 ```
 
 Notes
@@ -83,5 +89,5 @@ Installation
 Related efforts
 ============
 
-* [ruby-gumbo](https://github.com/galdor/ruby-gumbo#readme) - a ruby binding
+* [ruby-gumbo](https://github.com/nevir/ruby-gumbo#readme) - a ruby binding
 for the Gumbo HTML5 parser.

data/ext/nokogumboc/extconf.rb

@@ -1,7 +1,7 @@
 require 'mkmf'
 $CFLAGS += " -std=c99"
 
-if have_library('xml2', 'xmlNewDoc')
+if have_library('xml2', 'xmlNewDoc') 
   # libxml2 libraries from http://www.xmlsoft.org/
   pkg_config('libxml-2.0')
 
@@ -19,11 +19,6 @@ if have_library('xml2', 'xmlNewDoc')
 
     # if found, enable direct calls to Nokogiri (and libxml2)
     $CFLAGS += ' -DNGLIB' if find_header('nokogiri.h', nokogiri_ext)
-    
-    if File.exists?("/etc/gentoo-release")
-      # link to the library to prevent: nokogumbo.c:(.text+0x26a): undefined reference to `Nokogiri_wrap_xml_document'
-      $LDFLAGS += " -L#{nokogiri_ext} -l:nokogiri.so"
-    end
   end
 end
 
@@ -45,4 +40,21 @@ unless have_library('gumbo', 'gumbo_parse')
   end
 end
 
+# We use some Gumbo Internals, and not all distros ship the internal headers.
+header_typedefs = {
+  'error.h' => 'GumboErrorType',
+  'insertion_mode.h' => 'GumboInsertionMode',
+  'parser.h' => 'GumboParser',
+  'string_buffer.h' => 'GumboStringBuffer',
+  'token_type.h' => 'GumboTokenType',
+}
+
+header_typedefs.each_pair do |header, type|
+  unless find_type(type, header)
+    require 'fileutils'
+    FileUtils.cp Dir["#{rakehome}/gumbo-parser/src/#{header}"],
+      "#{rakehome}/ext/nokogumboc"
+  end
+end
+
 create_makefile('nokogumboc')

data/ext/nokogumboc/nokogumbo.c

@@ -19,10 +19,13 @@
 //
 
 #include <ruby.h>
-#include <gumbo.h>
+#include "gumbo.h"
+#include "error.h"
+#include "parser.h"
 
 // class constants
 static VALUE Document;
+static VALUE XMLSyntaxError;
 
 #ifdef NGLIB
 #include <nokogiri.h>
@@ -82,8 +85,11 @@ static VALUE xmlNewDoc(char* version) {
 }
 #endif
 
-// Build a Nokogiri Element for a given GumboElement (recursively)
-static xmlNodePtr walk_tree(xmlDocPtr document, GumboElement *node) {
+// Build a xmlNodePtr for a given GumboNode (recursively)
+static xmlNodePtr walk_tree(xmlDocPtr document, GumboNode *node);
+
+// Build a xmlNodePtr for a given GumboElement (recursively)
+static xmlNodePtr walk_element(xmlDocPtr document, GumboElement *node) {
   // determine tag name for a given node
   xmlNodePtr element;
   if (node->tag != GUMBO_TAG_UNKNOWN) {
@@ -151,55 +157,108 @@ static xmlNodePtr walk_tree(xmlDocPtr document, GumboElement *node) {
   // add in the children
   GumboVector* children = &node->children;
   for (int i=0; i < children->length; i++) {
-    GumboNode* child = children->data[i];
-
-    xmlNodePtr node = NIL;
-
-    switch (child->type) {
-      case GUMBO_NODE_ELEMENT:
-        node = walk_tree(document, &child->v.element);
-        break;
-      case GUMBO_NODE_WHITESPACE:
-      case GUMBO_NODE_TEXT:
-        node = xmlNewDocText(document, CONST_CAST child->v.text.text);
-        break;
-      case GUMBO_NODE_CDATA:
-        node = xmlNewCDataBlock(document, 
-          CONST_CAST child->v.text.original_text.data,
-          (int) child->v.text.original_text.length);
-        break;
-      case GUMBO_NODE_COMMENT:
-        node = xmlNewDocComment(document, CONST_CAST child->v.text.text);
-        break;
-      case GUMBO_NODE_DOCUMENT:
-        break; // should never happen -- ignore
-    }
-
+    xmlNodePtr node = walk_tree(document, children->data[i]);
     if (node) xmlAddChild(element, node);
   }
 
   return element;
 }
 
+static xmlNodePtr walk_tree(xmlDocPtr document, GumboNode *node) {
+  switch (node->type) {
+    case GUMBO_NODE_DOCUMENT:
+      return NIL;
+    case GUMBO_NODE_ELEMENT:
+    case GUMBO_NODE_TEMPLATE:
+      return walk_element(document, &node->v.element);
+    case GUMBO_NODE_TEXT:
+    case GUMBO_NODE_WHITESPACE:
+      return xmlNewDocText(document, CONST_CAST node->v.text.text);
+    case GUMBO_NODE_CDATA:
+      return xmlNewCDataBlock(document,
+        CONST_CAST node->v.text.original_text.data,
+        (int) node->v.text.original_text.length);
+    case GUMBO_NODE_COMMENT:
+      return xmlNewDocComment(document, CONST_CAST node->v.text.text);
+  }
+}
+
 // Parse a string using gumbo_parse into a Nokogiri document
-static VALUE parse(VALUE self, VALUE string) {
-  GumboOutput *output = gumbo_parse_with_options(
-    &kGumboDefaultOptions, RSTRING_PTR(string),
-    (size_t) RSTRING_LEN(string)
-  );
+static VALUE parse(VALUE self, VALUE string, VALUE max_parse_errors) {
+  GumboOptions options;
+  memcpy(&options, &kGumboDefaultOptions, sizeof options);
+  options.max_errors = NUM2INT(max_parse_errors);
+
+  const char *input = RSTRING_PTR(string);
+  size_t input_len = RSTRING_LEN(string);
+  GumboOutput *output = gumbo_parse_with_options(&options, input, input_len);
   xmlDocPtr doc = xmlNewDoc(CONST_CAST "1.0");
-  xmlNodePtr root = walk_tree(doc, &output->root->v.element);
-  xmlDocSetRootElement(doc, root);
+#ifdef NGLIB
+  doc->type = XML_HTML_DOCUMENT_NODE;
+#endif
   if (output->document->v.document.has_doctype) {
+    const char *name   = output->document->v.document.name;
     const char *public = output->document->v.document.public_identifier;
     const char *system = output->document->v.document.system_identifier;
-    xmlCreateIntSubset(doc, CONST_CAST "html",
-      (strlen(public) ? CONST_CAST public : NIL),
-      (strlen(system) ? CONST_CAST system : NIL));
+    xmlCreateIntSubset(doc, CONST_CAST name,
+      (public[0] ? CONST_CAST public : NIL),
+      (system[0] ? CONST_CAST system : NIL));
   }
-  gumbo_destroy_output(&kGumboDefaultOptions, output);
 
-  return Nokogiri_wrap_xml_document(Document, doc);
+  GumboVector *children = &output->document->v.document.children;
+  for (int i=0; i < children->length; i++) {
+    GumboNode *child = children->data[i];
+    xmlNodePtr node = walk_tree(doc, child);
+    if (node) {
+      if (child == output->root)
+        xmlDocSetRootElement(doc, node);
+      else
+        xmlAddChild((xmlNodePtr)doc, node);
+    }
+  }
+
+  VALUE rdoc = Nokogiri_wrap_xml_document(Document, doc);
+
+  // Add parse errors to rdoc.
+  if (output->errors.length) {
+    GumboVector *errors = &output->errors;
+    GumboParser parser = { ._options = &options };
+    GumboStringBuffer msg;
+    VALUE rerrors = rb_ary_new2(errors->length);
+
+    gumbo_string_buffer_init(&parser, &msg);
+    for (int i=0; i < errors->length; i++) {
+      GumboError *err = errors->data[i];
+      gumbo_string_buffer_clear(&parser, &msg);
+      // Work around bug in gumbo_caret_diagnostic_to_string.
+      // See https://github.com/google/gumbo-parser/pull/371
+      // The bug occurs when the error starts with a newline (unless it's the
+      // first character in the input--but that shouldn't cause an error in
+      // the first place.
+      if (*err->original_text == '\n' && err->original_text != input)
+        --err->original_text;
+      gumbo_caret_diagnostic_to_string(&parser, err, input, &msg);
+      VALUE err_str = rb_str_new(msg.data, msg.length);
+      VALUE syntax_error = rb_class_new_instance(1, &err_str, XMLSyntaxError);
+      rb_iv_set(syntax_error, "@domain", INT2NUM(1)); // XML_FROM_PARSER
+      rb_iv_set(syntax_error, "@code", INT2NUM(1));   // XML_ERR_INTERNAL_ERROR
+      rb_iv_set(syntax_error, "@level", INT2NUM(2));  // XML_ERR_ERROR
+      rb_iv_set(syntax_error, "@file", Qnil);
+      rb_iv_set(syntax_error, "@line", INT2NUM(err->position.line));
+      rb_iv_set(syntax_error, "@str1", Qnil);
+      rb_iv_set(syntax_error, "@str2", Qnil);
+      rb_iv_set(syntax_error, "@str3", Qnil);
+      rb_iv_set(syntax_error, "@int1", INT2NUM(err->type));
+      rb_iv_set(syntax_error, "@column", INT2NUM(err->position.column));
+      rb_ary_push(rerrors, syntax_error);
+    }
+    rb_iv_set(rdoc, "@errors", rerrors);
+    gumbo_string_buffer_destroy(&parser, &msg);
+  }
+
+  gumbo_destroy_output(&options, output);
+
+  return rdoc;
 }
 
 // Initialize the Nokogumbo class and fetch constants we will use later
@@ -211,10 +270,11 @@ void Init_nokogumboc() {
   VALUE Nokogiri = rb_const_get(rb_cObject, rb_intern("Nokogiri"));
   VALUE HTML = rb_const_get(Nokogiri, rb_intern("HTML"));
   Document = rb_const_get(HTML, rb_intern("Document"));
+  VALUE XML = rb_const_get(Nokogiri, rb_intern("XML"));
+  XMLSyntaxError = rb_const_get(XML, rb_intern("SyntaxError"));
 
 #ifndef NGLIB
   // more class constants
-  VALUE XML = rb_const_get(Nokogiri, rb_intern("XML"));
   Element = rb_const_get(XML, rb_intern("Element"));
   Text = rb_const_get(XML, rb_intern("Text"));
   CDATA = rb_const_get(XML, rb_intern("CDATA"));
@@ -223,7 +283,7 @@ void Init_nokogumboc() {
   // interned symbols
   new = rb_intern("new");
   set_attribute = rb_intern("set_attribute");
-  add_child = rb_intern("add_child");
+  add_child = rb_intern("add_child_node_and_reparent_attrs");
   internal_subset = rb_intern("internal_subset");
   remove_ = rb_intern("remove");
   create_internal_subset = rb_intern("create_internal_subset");
@@ -231,5 +291,5 @@ void Init_nokogumboc() {
 
   // define Nokogumbo class with a singleton parse method
   VALUE Gumbo = rb_define_class("Nokogumbo", rb_cObject);
-  rb_define_singleton_method(Gumbo, "parse", parse, 1);
+  rb_define_singleton_method(Gumbo, "parse", parse, 2);
 }

data/gumbo-parser/src/attribute.c

@@ -27,7 +27,7 @@ struct GumboInternalParser;
 
 GumboAttribute* gumbo_get_attribute(
     const GumboVector* attributes, const char* name) {
-  for (int i = 0; i < attributes->length; ++i) {
+  for (unsigned int i = 0; i < attributes->length; ++i) {
     GumboAttribute* attr = attributes->data[i];
     if (!strcasecmp(attr->name, name)) {
       return attr;

data/gumbo-parser/src/char_ref.c

@@ -30,7 +30,7 @@
 #include <ctype.h>
 #include <stddef.h>
 #include <stdio.h>
-#include <string.h>     // Only for debug assertions at present.
+#include <string.h>  // Only for debug assertions at present.
 
 #include "error.h"
 #include "string_piece.h"
@@ -49,44 +49,18 @@ typedef struct {
   int to_char;
 } CharReplacement;
 
-static const CharReplacement kCharReplacements[] = {
-  { 0x00, 0xfffd },
-  { 0x0d, 0x000d },
-  { 0x80, 0x20ac },
-  { 0x81, 0x0081 },
-  { 0x82, 0x201A },
-  { 0x83, 0x0192 },
-  { 0x84, 0x201E },
-  { 0x85, 0x2026 },
-  { 0x86, 0x2020 },
-  { 0x87, 0x2021 },
-  { 0x88, 0x02C6 },
-  { 0x89, 0x2030 },
-  { 0x8A, 0x0160 },
-  { 0x8B, 0x2039 },
-  { 0x8C, 0x0152 },
-  { 0x8D, 0x008D },
-  { 0x8E, 0x017D },
-  { 0x8F, 0x008F },
-  { 0x90, 0x0090 },
-  { 0x91, 0x2018 },
-  { 0x92, 0x2019 },
-  { 0x93, 0x201C },
-  { 0x94, 0x201D },
-  { 0x95, 0x2022 },
-  { 0x96, 0x2013 },
-  { 0x97, 0x2014 },
-  { 0x98, 0x02DC },
-  { 0x99, 0x2122 },
-  { 0x9A, 0x0161 },
-  { 0x9B, 0x203A },
-  { 0x9C, 0x0153 },
-  { 0x9D, 0x009D },
-  { 0x9E, 0x017E },
-  { 0x9F, 0x0178 },
-  // Terminator.
-  { -1, -1 }
-};
+static const CharReplacement kCharReplacements[] = {{0x00, 0xfffd},
+    {0x0d, 0x000d}, {0x80, 0x20ac}, {0x81, 0x0081}, {0x82, 0x201A},
+    {0x83, 0x0192}, {0x84, 0x201E}, {0x85, 0x2026}, {0x86, 0x2020},
+    {0x87, 0x2021}, {0x88, 0x02C6}, {0x89, 0x2030}, {0x8A, 0x0160},
+    {0x8B, 0x2039}, {0x8C, 0x0152}, {0x8D, 0x008D}, {0x8E, 0x017D},
+    {0x8F, 0x008F}, {0x90, 0x0090}, {0x91, 0x2018}, {0x92, 0x2019},
+    {0x93, 0x201C}, {0x94, 0x201D}, {0x95, 0x2022}, {0x96, 0x2013},
+    {0x97, 0x2014}, {0x98, 0x02DC}, {0x99, 0x2122}, {0x9A, 0x0161},
+    {0x9B, 0x203A}, {0x9C, 0x0153}, {0x9D, 0x009D}, {0x9E, 0x017E},
+    {0x9F, 0x0178},
+    // Terminator.
+    {-1, -1}};
 
 static int parse_digit(int c, bool allow_hex) {
   if (c >= '0' && c <= '9') {
@@ -111,9 +85,8 @@ static void add_no_digit_error(
   error->type = GUMBO_ERR_NUMERIC_CHAR_REF_NO_DIGITS;
 }
 
-static void add_codepoint_error(
-    struct GumboInternalParser* parser, Utf8Iterator* input,
-    GumboErrorType type, int codepoint) {
+static void add_codepoint_error(struct GumboInternalParser* parser,
+    Utf8Iterator* input, GumboErrorType type, int codepoint) {
   GumboError* error = gumbo_add_error(parser);
   if (!error) {
     return;
@@ -123,9 +96,8 @@ static void add_codepoint_error(
   error->v.codepoint = codepoint;
 }
 
-static void add_named_reference_error(
-    struct GumboInternalParser* parser, Utf8Iterator* input,
-    GumboErrorType type, GumboStringPiece text) {
+static void add_named_reference_error(struct GumboInternalParser* parser,
+    Utf8Iterator* input, GumboErrorType type, GumboStringPiece text) {
   GumboError* error = gumbo_add_error(parser);
   if (!error) {
     return;
@@ -211,8 +183,7 @@ static bool maybe_add_invalid_named_reference(
   // worry about consuming characters.
   const char* start = utf8iterator_get_char_pointer(input);
   int c = utf8iterator_current(input);
-  while ((c >= 'a' && c <= 'z') ||
-         (c >= 'A' && c <= 'Z') ||
+  while ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
          (c >= '0' && c <= '9')) {
     utf8iterator_next(input);
     c = utf8iterator_current(input);
@@ -228,12 +199,11 @@ static bool maybe_add_invalid_named_reference(
   return true;
 }
 
-
 #line 2465 "char_ref.rl"
 
+// clang-format off
 
-
-#line 237 "char_ref.c"
+#line 238 "char_ref.c"
 static const short _char_ref_actions[] = {
 	0, 1, 0, 1, 1, 1, 2, 1, 
 	3, 1, 4, 1, 5, 1, 6, 1, 
@@ -13960,17 +13930,15 @@ static const short _char_ref_eof_trans[] = {
 };
 
 static const int char_ref_start = 7623;
-static const int char_ref_first_final = 7623;
-static const int char_ref_error = 0;
 
 static const int char_ref_en_valid_named_ref = 7623;
 
 
-#line 2468 "char_ref.rl"
+#line 2469 "char_ref.rl"
+// clang-format on
 
-static bool consume_named_ref(
-    struct GumboInternalParser* parser, Utf8Iterator* input, bool is_in_attribute,
-    OneOrTwoCodepoints* output) {
+static bool consume_named_ref(struct GumboInternalParser* parser,
+    Utf8Iterator* input, bool is_in_attribute, OneOrTwoCodepoints* output) {
   assert(output->first == kGumboNoChar);
   const char* p = utf8iterator_get_char_pointer(input);
   const char* pe = utf8iterator_get_end_pointer(input);
@@ -13979,8 +13947,9 @@ static bool consume_named_ref(
   const char *ts, *start;
   int cs, act;
 
+// clang-format off
   
-#line 13984 "char_ref.c"
+#line 13985 "char_ref.c"
 	{
 	cs = char_ref_start;
 	ts = 0;
@@ -13988,14 +13957,15 @@ static bool consume_named_ref(
 	act = 0;
 	}
 
-#line 2481 "char_ref.rl"
+#line 2484 "char_ref.rl"
   // Avoid unused variable warnings.
   (void) act;
   (void) ts;
+  (void) char_ref_en_valid_named_ref;
 
   start = p;
   
-#line 13999 "char_ref.c"
+#line 14001 "char_ref.c"
 	{
 	int _slen;
 	int _trans;
@@ -14017,7 +13987,7 @@ _resume:
 #line 1 "NONE"
 	{ts = p;}
 	break;
-#line 14021 "char_ref.c"
+#line 14023 "char_ref.c"
 		}
 	}
 
@@ -23000,7 +22970,7 @@ _eof_trans:
 #line 2273 "char_ref.rl"
 	{{p = ((te))-1;}{ output->first = 0xd7; {p++; goto _out; } }}
 	break;
-#line 23004 "char_ref.c"
+#line 23006 "char_ref.c"
 		}
 	}
 
@@ -23013,7 +22983,7 @@ _again:
 #line 1 "NONE"
 	{ts = 0;}
 	break;
-#line 23017 "char_ref.c"
+#line 23019 "char_ref.c"
 		}
 	}
 
@@ -23033,7 +23003,8 @@ _again:
 	_out: {}
 	}
 
-#line 2487 "char_ref.rl"
+#line 2491 "char_ref.rl"
+  // clang-format on
 
   if (cs >= 7623) {
     assert(output->first != kGumboNoChar);
@@ -23067,10 +23038,9 @@ _again:
   }
 }
 
-bool consume_char_ref(
-    struct GumboInternalParser* parser, struct GumboInternalUtf8Iterator* input,
-    int additional_allowed_char, bool is_in_attribute,
-    OneOrTwoCodepoints* output) {
+bool consume_char_ref(struct GumboInternalParser* parser,
+    struct GumboInternalUtf8Iterator* input, int additional_allowed_char,
+    bool is_in_attribute, OneOrTwoCodepoints* output) {
   utf8iterator_mark(input);
   utf8iterator_next(input);
   int c = utf8iterator_current(input);