nokogumbo 1.4.1 → 1.4.2

data/gumbo-parser/src/string_buffer.c

@@ -26,9 +26,7 @@
 
 struct GumboInternalParser;
 
-// Size chosen via statistical analysis of ~60K websites.
-// 99% of text nodes and 98% of attribute names/values fit in this initial size.
-static const size_t kDefaultStringBufferSize = 5;
+static const size_t kDefaultStringBufferSize = 10;
 
 static void maybe_resize_string_buffer(
     struct GumboInternalParser* parser, size_t additional_chars,
@@ -102,11 +100,6 @@ char* gumbo_string_buffer_to_string(
   return buffer;
 }
 
-void gumbo_string_buffer_clear(
-    struct GumboInternalParser* parser, GumboStringBuffer* input) {
-  input->length = 0;
-}
-
 void gumbo_string_buffer_destroy(
     struct GumboInternalParser* parser, GumboStringBuffer* buffer) {
   gumbo_parser_deallocate(parser, buffer->data);

data/gumbo-parser/src/string_buffer.h

@@ -70,11 +70,6 @@ void gumbo_string_buffer_append_string(
 char* gumbo_string_buffer_to_string(
     struct GumboInternalParser* parser, GumboStringBuffer* input);
 
-// Reinitialize this string buffer.  This clears it by setting length=0.  It
-// does not zero out the buffer itself.
-void gumbo_string_buffer_clear(
-    struct GumboInternalParser* parser, GumboStringBuffer* input);
-
 // Deallocates this GumboStringBuffer.
 void gumbo_string_buffer_destroy(
     struct GumboInternalParser* parser, GumboStringBuffer* buffer);

data/gumbo-parser/src/tag.c

@@ -18,25 +18,172 @@
 
 #include <assert.h>
 #include <ctype.h>
-#include <string.h>
+#include <strings.h>    // For strcasecmp.
 
+// NOTE(jdtang): Keep this in sync with the GumboTag enum in the header.
+// TODO(jdtang): Investigate whether there're efficiency benefits to putting the
+// most common tag names first, or to putting them in alphabetical order and
+// using a binary search.
 const char* kGumboTagNames[] = {
-# include "tag_strings.h"
+  "html",
+  "head",
+  "title",
+  "base",
+  "link",
+  "meta",
+  "style",
+  "script",
+  "noscript",
+  "template",
+  "body",
+  "article",
+  "section",
+  "nav",
+  "aside",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "hgroup",
+  "header",
+  "footer",
+  "address",
+  "p",
+  "hr",
+  "pre",
+  "blockquote",
+  "ol",
+  "ul",
+  "li",
+  "dl",
+  "dt",
+  "dd",
+  "figure",
+  "figcaption",
+  "main",
+  "div",
+  "a",
+  "em",
+  "strong",
+  "small",
+  "s",
+  "cite",
+  "q",
+  "dfn",
+  "abbr",
+  "data",
+  "time",
+  "code",
+  "var",
+  "samp",
+  "kbd",
+  "sub",
+  "sup",
+  "i",
+  "b",
+  "u",
+  "mark",
+  "ruby",
+  "rt",
+  "rp",
+  "bdi",
+  "bdo",
+  "span",
+  "br",
+  "wbr",
+  "ins",
+  "del",
+  "image",
+  "img",
+  "iframe",
+  "embed",
+  "object",
+  "param",
+  "video",
+  "audio",
+  "source",
+  "track",
+  "canvas",
+  "map",
+  "area",
+  "math",
+  "mi",
+  "mo",
+  "mn",
+  "ms",
+  "mtext",
+  "mglyph",
+  "malignmark",
+  "annotation-xml",
+  "svg",
+  "foreignobject",
+  "desc",
+  "table",
+  "caption",
+  "colgroup",
+  "col",
+  "tbody",
+  "thead",
+  "tfoot",
+  "tr",
+  "td",
+  "th",
+  "form",
+  "fieldset",
+  "legend",
+  "label",
+  "input",
+  "button",
+  "select",
+  "datalist",
+  "optgroup",
+  "option",
+  "textarea",
+  "keygen",
+  "output",
+  "progress",
+  "meter",
+  "details",
+  "summary",
+  "menu",
+  "menuitem",
+  "applet",
+  "acronym",
+  "bgsound",
+  "dir",
+  "frame",
+  "frameset",
+  "noframes",
+  "isindex",
+  "listing",
+  "xmp",
+  "nextid",
+  "noembed",
+  "plaintext",
+  "rb",
+  "strike",
+  "basefont",
+  "big",
+  "blink",
+  "center",
+  "font",
+  "marquee",
+  "multicol",
+  "nobr",
+  "spacer",
+  "tt",
   "",                   // TAG_UNKNOWN
   "",                   // TAG_LAST
 };
 
-static const unsigned char kGumboTagSizes[] = {
-# include "tag_sizes.h"
-  0, // TAG_UNKNOWN
-  0, // TAG_LAST
-};
-
 const char* gumbo_normalized_tagname(GumboTag tag) {
   assert(tag <= GUMBO_TAG_LAST);
   return kGumboTagNames[tag];
 }
 
+// TODO(jdtang): Add test for this.
 void gumbo_tag_from_original_text(GumboStringPiece* text) {
   if (text->data == NULL) {
     return;
@@ -65,34 +212,14 @@ void gumbo_tag_from_original_text(GumboStringPiece* text) {
   }
 }
 
-static int
-case_memcmp(const char *s1, const char *s2, unsigned int n)
-{
-	while (n--) {
-		unsigned char c1 = tolower(*s1++);
-		unsigned char c2 = tolower(*s2++);
-		if (c1 != c2)
-			return (int)c1 - (int)c2;
-	}
-	return 0;
-}
-
-#include "tag_gperf.h"
-#define TAG_MAP_SIZE (sizeof(kGumboTagMap)/sizeof(kGumboTagMap[0]))
-
-GumboTag gumbo_tagn_enum(const char* tagname, unsigned int length) {
-  if (length) {
-    unsigned int key = tag_hash(tagname, length);
-    if (key < TAG_MAP_SIZE) {
-      GumboTag tag = kGumboTagMap[key];
-      if (length == kGumboTagSizes[(int)tag] &&
-          !case_memcmp(tagname, kGumboTagNames[(int)tag], length))
-        return tag;
+GumboTag gumbo_tag_enum(const char* tagname) {
+  for (int i = 0; i < GUMBO_TAG_LAST; ++i) {
+    // TODO(jdtang): strcasecmp is non-portable, so if we want to support
+    // non-GCC compilers, we'll need some #ifdef magic.  This source already has
+    // pretty significant issues with MSVC6 anyway.
+    if (strcasecmp(tagname, kGumboTagNames[i]) == 0) {
+      return i;
     }
   }
   return GUMBO_TAG_UNKNOWN;
 }
-
-GumboTag gumbo_tag_enum(const char* tagname) {
-  return gumbo_tagn_enum(tagname, strlen(tagname));
-}

data/gumbo-parser/src/tokenizer.c

@@ -356,10 +356,12 @@ static void clear_temporary_buffer(GumboParser* parser) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   assert(!tokenizer->_temporary_buffer_emit);
   utf8iterator_mark(&tokenizer->_input);
-  gumbo_string_buffer_clear(parser, &tokenizer->_temporary_buffer);
+  gumbo_string_buffer_destroy(parser, &tokenizer->_temporary_buffer);
+  gumbo_string_buffer_init(parser, &tokenizer->_temporary_buffer);
   // The temporary buffer and script data buffer are the same object in the
   // spec, so the script data buffer should be cleared as well.
-  gumbo_string_buffer_clear(parser, &tokenizer->_script_data_buffer);
+  gumbo_string_buffer_destroy(parser, &tokenizer->_script_data_buffer);
+  gumbo_string_buffer_init(parser, &tokenizer->_script_data_buffer);
 }
 
 // Appends a codepoint to the temporary buffer.
@@ -695,11 +697,7 @@ static void start_new_tag(GumboParser* parser, bool is_start_tag) {
   gumbo_string_buffer_append_codepoint(parser, c, &tag_state->_buffer);
 
   assert(tag_state->_attributes.data == NULL);
-  // Initial size chosen by statistical analysis of a corpus of 60k webpages.
-  // 99.5% of elements have 0 attributes, 93% of the remainder have 1.  These
-  // numbers are a bit higher for more modern websites (eg. ~45% = 0, ~40% = 1
-  // for the HTML5 Spec), but still have basically 99% of nodes with <= 2 attrs.
-  gumbo_vector_init(parser, 1, &tag_state->_attributes);
+  gumbo_vector_init(parser, 4, &tag_state->_attributes);
   tag_state->_drop_next_attr_value = false;
   tag_state->_is_start_tag = is_start_tag;
   tag_state->_is_self_closing = false;
@@ -753,9 +751,11 @@ static void finish_tag_name(GumboParser* parser) {
   GumboTokenizerState* tokenizer = parser->_tokenizer_state;
   GumboTagState* tag_state = &tokenizer->_tag_state;
 
-  tag_state->_tag = gumbo_tagn_enum(
-		tag_state->_buffer.data, tag_state->_buffer.length);
+  const char* temp;
+  copy_over_tag_buffer(parser, &temp);
+  tag_state->_tag = gumbo_tag_enum(temp);
   reinitialize_tag_buffer(parser);
+  gumbo_parser_deallocate(parser, (void*) temp);
 }
 
 // Adds an ERR_DUPLICATE_ATTR parse error to the parser's error struct.
@@ -841,9 +841,13 @@ static void finish_attribute_value(GumboParser* parser) {
 static bool is_appropriate_end_tag(GumboParser* parser) {
   GumboTagState* tag_state = &parser->_tokenizer_state->_tag_state;
   assert(!tag_state->_is_start_tag);
+  // Null terminate the current string buffer, so it can be passed to
+  // gumbo_tag_enum, but don't increment the length in case we need to dump the
+  // buffer as character tokens.
+  gumbo_string_buffer_append_codepoint(parser, '\0', &tag_state->_buffer);
+  --tag_state->_buffer.length;
   return tag_state->_last_start_tag != GUMBO_TAG_LAST &&
-      tag_state->_last_start_tag ==
-	  gumbo_tagn_enum(tag_state->_buffer.data, tag_state->_buffer.length);
+      tag_state->_last_start_tag == gumbo_tag_enum(tag_state->_buffer.data);
 }
 
 void gumbo_tokenizer_state_init(
@@ -1593,7 +1597,8 @@ static StateResult handle_script_double_escaped_lt_state(
     int c, GumboToken* output) {
   if (c == '/') {
     gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END);
-    gumbo_string_buffer_clear(parser, &tokenizer->_script_data_buffer);
+    gumbo_string_buffer_destroy(parser, &tokenizer->_script_data_buffer);
+    gumbo_string_buffer_init(parser, &tokenizer->_script_data_buffer);
     return emit_current_char(parser, output);
   } else {
     gumbo_tokenizer_set_state(parser, GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED);
@@ -2819,7 +2824,7 @@ static StateResult handle_cdata_state(
     tokenizer->_reconsume_current_input = true;
     reset_token_start_point(tokenizer);
     gumbo_tokenizer_set_state(parser, GUMBO_LEX_DATA);
-    tokenizer->_is_in_cdata = false;
+    tokenizer->_is_in_cdata = true;
     return NEXT_CHAR;
   } else {
     return emit_current_char(parser, output);

data/gumbo-parser/src/vector.c

@@ -81,7 +81,7 @@ void* gumbo_vector_pop(
   return vector->data[--vector->length];
 }
 
-int gumbo_vector_index_of(GumboVector* vector, const void* element) {
+int gumbo_vector_index_of(GumboVector* vector, void* element) {
   for (int i = 0; i < vector->length; ++i) {
     if (vector->data[i] == element) {
       return i;

data/test-nokogumbo.rb

@@ -99,7 +99,7 @@ class TestNokogumbo < Minitest::Test
     assert_equal ["xlink:href", "xmlns:xlink"], a.attributes.keys.sort
   end
 
-  def test_template
+  def x_test_template # future
     source = <<-EOF.gsub(/^ {6}/, '')
       <template id="productrow">
         <tr>

metadata

@@ -1,32 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: nokogumbo
 version: !ruby/object:Gem::Version
-  prerelease: 
-  version: 1.4.1
+  version: 1.4.2
 platform: ruby
 authors:
 - Sam Ruby
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-13 00:00:00.000000000 Z
+date: 2015-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  version_requirements: !ruby/object:Gem::Requirement
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
   type: :runtime
-  name: nokogiri
   prerelease: false
-  requirement: !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 description: Nokogumbo allows a Ruby program to invoke the Gumbo HTML5 parser and
   access the result as a Nokogiri parsed document.
 email: rubys@intertwingly.net
@@ -35,11 +32,10 @@ extensions:
 - ext/nokogumboc/extconf.rb
 extra_rdoc_files: []
 files:
-- ext/nokogumboc/extconf.rb
-- ext/nokogumboc/nokogumbo.c
-- lib/nokogumbo.rb
 - LICENSE.txt
 - README.md
+- ext/nokogumboc/extconf.rb
+- ext/nokogumboc/nokogumbo.c
 - gumbo-parser/src/attribute.c
 - gumbo-parser/src/attribute.h
 - gumbo-parser/src/char_ref.c
@@ -56,11 +52,6 @@ files:
 - gumbo-parser/src/string_piece.c
 - gumbo-parser/src/string_piece.h
 - gumbo-parser/src/tag.c
-- gumbo-parser/src/tag.in
-- gumbo-parser/src/tag_enum.h
-- gumbo-parser/src/tag_gperf.h
-- gumbo-parser/src/tag_sizes.h
-- gumbo-parser/src/tag_strings.h
 - gumbo-parser/src/token_type.h
 - gumbo-parser/src/tokenizer.c
 - gumbo-parser/src/tokenizer.h
@@ -72,30 +63,30 @@ files:
 - gumbo-parser/src/vector.c
 - gumbo-parser/src/vector.h
 - gumbo-parser/visualc/include/strings.h
+- lib/nokogumbo.rb
 - test-nokogumbo.rb
 homepage: https://github.com/rubys/nokogumbo/#readme
 licenses:
 - Apache 2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23.2
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Nokogiri interface to the Gumbo HTML5 parser
 test_files: []