nokogumbo 1.5.0 → 2.0.0.pre.alpha

data/gumbo-parser/src/util.c

@@ -1,58 +1,68 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
+/*
+ Copyright 2017-2018 Craig Barnes.
+ Copyright 2010 Google Inc.
 
-#include "util.h"
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
 
-#include <assert.h>
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <strings.h>
-#include <stdarg.h>
-#include <stdio.h>
-
+#include "util.h"
 #include "gumbo.h"
-#include "parser.h"
 
-// TODO(jdtang): This should be elsewhere, but there's no .c file for
-// SourcePositions and yet the constant needs some linkage, so this is as good
-// as any.
-const GumboSourcePosition kGumboEmptySourcePosition = {0, 0, 0};
+void* gumbo_alloc(size_t size) {
+  void* ptr = malloc(size);
+  if (unlikely(ptr == NULL)) {
+    perror(__func__);
+    abort();
+  }
+  return ptr;
+}
 
-void* gumbo_parser_allocate(GumboParser* parser, size_t num_bytes) {
-  return parser->_options->allocator(parser->_options->userdata, num_bytes);
+void* gumbo_realloc(void* ptr, size_t size) {
+  ptr = realloc(ptr, size);
+  if (unlikely(ptr == NULL)) {
+    perror(__func__);
+    abort();
+  }
+  return ptr;
 }
 
-void gumbo_parser_deallocate(GumboParser* parser, void* ptr) {
-  parser->_options->deallocator(parser->_options->userdata, ptr);
+void gumbo_free(void* ptr) {
+  free(ptr);
 }
 
-char* gumbo_copy_stringz(GumboParser* parser, const char* str) {
-  char* buffer = gumbo_parser_allocate(parser, strlen(str) + 1);
-  strcpy(buffer, str);
-  return buffer;
+char* gumbo_strdup(const char* str) {
+  const size_t size = strlen(str) + 1;
+  // The strdup(3) function isn't available in strict "-std=c99" mode
+  // (it's part of POSIX, not C99), so use malloc(3) and memcpy(3)
+  // instead:
+  char* buffer = gumbo_alloc(size);
+  return memcpy(buffer, str, size);
 }
 
-// Debug function to trace operation of the parser.  Pass --copts=-DGUMBO_DEBUG
-// to use.
-void gumbo_debug(const char* format, ...) {
 #ifdef GUMBO_DEBUG
+#include <stdarg.h>
+// Debug function to trace operation of the parser
+// (define GUMBO_DEBUG to use).
+void gumbo_debug(const char* format, ...) {
   va_list args;
   va_start(args, format);
   vprintf(format, args);
   va_end(args);
   fflush(stdout);
-#endif
 }
+#else
+void gumbo_debug(const char* UNUSED_ARG(format), ...) {}
+#endif

data/gumbo-parser/src/util.h

@@ -1,60 +1,30 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-//
-// This contains some utility functions that didn't fit into any of the other
-// headers.
-
 #ifndef GUMBO_UTIL_H_
 #define GUMBO_UTIL_H_
-#ifdef _MSC_VER
-#define _CRT_SECURE_NO_WARNINGS
-#endif
+
 #include <stdbool.h>
 #include <stddef.h>
+#include "macros.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-// Forward declaration since it's passed into some of the functions in this
-// header.
-struct GumboInternalParser;
-
 // Utility function for allocating & copying a null-terminated string into a
-// freshly-allocated buffer.  This is necessary for proper memory management; we
+// freshly-allocated buffer. This is necessary for proper memory management; we
 // have the convention that all const char* in parse tree structures are
 // freshly-allocated, so if we didn't copy, we'd try to delete a literal string
 // when the parse tree is destroyed.
-char* gumbo_copy_stringz(struct GumboInternalParser* parser, const char* str);
-
-// Allocate a chunk of memory, using the allocator specified in the Parser's
-// config options.
-void* gumbo_parser_allocate(
-    struct GumboInternalParser* parser, size_t num_bytes);
+char* gumbo_strdup(const char* str) XMALLOC NONNULL_ARGS;
 
-// Deallocate a chunk of memory, using the deallocator specified in the Parser's
-// config options.
-void gumbo_parser_deallocate(struct GumboInternalParser* parser, void* ptr);
+void* gumbo_alloc(size_t size) XMALLOC;
+void* gumbo_realloc(void* ptr, size_t size) RETURNS_NONNULL;
+void gumbo_free(void* ptr);
 
-// Debug wrapper for printf, to make it easier to turn off debugging info when
-// required.
-void gumbo_debug(const char* format, ...);
+// Debug wrapper for printf
+void gumbo_debug(const char* format, ...) PRINTF(1);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // GUMBO_UTIL_H_
+#endif // GUMBO_UTIL_H_

data/gumbo-parser/src/vector.c

@@ -1,81 +1,64 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
+/*
+ Copyright 2018 Craig Barnes.
+ Copyright 2010 Google Inc.
 
-#include "vector.h"
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
 
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>
-#include <strings.h>
-
+#include "vector.h"
 #include "util.h"
 
-struct GumboInternalParser;
-
-const GumboVector kGumboEmptyVector = {NULL, 0, 0};
-
-void gumbo_vector_init(struct GumboInternalParser* parser,
-    size_t initial_capacity, GumboVector* vector) {
+void gumbo_vector_init(unsigned int initial_capacity, GumboVector* vector) {
   vector->length = 0;
   vector->capacity = initial_capacity;
   if (initial_capacity > 0) {
-    vector->data =
-        gumbo_parser_allocate(parser, sizeof(void*) * initial_capacity);
+    vector->data = gumbo_alloc(sizeof(void*) * initial_capacity);
   } else {
     vector->data = NULL;
   }
 }
 
-void gumbo_vector_destroy(
-    struct GumboInternalParser* parser, GumboVector* vector) {
+void gumbo_vector_destroy(GumboVector* vector) {
   if (vector->capacity > 0) {
-    gumbo_parser_deallocate(parser, vector->data);
+    gumbo_free(vector->data);
   }
 }
 
-static void enlarge_vector_if_full(
-    struct GumboInternalParser* parser, GumboVector* vector) {
+static void enlarge_vector_if_full(GumboVector* vector) {
   if (vector->length >= vector->capacity) {
     if (vector->capacity) {
-      size_t old_num_bytes = sizeof(void*) * vector->capacity;
       vector->capacity *= 2;
       size_t num_bytes = sizeof(void*) * vector->capacity;
-      void** temp = gumbo_parser_allocate(parser, num_bytes);
-      memcpy(temp, vector->data, old_num_bytes);
-      gumbo_parser_deallocate(parser, vector->data);
-      vector->data = temp;
+      vector->data = gumbo_realloc(vector->data, num_bytes);
     } else {
       // 0-capacity vector; no previous array to deallocate.
       vector->capacity = 2;
-      vector->data =
-          gumbo_parser_allocate(parser, sizeof(void*) * vector->capacity);
+      vector->data = gumbo_alloc(sizeof(void*) * vector->capacity);
     }
   }
 }
 
-void gumbo_vector_add(
-    struct GumboInternalParser* parser, void* element, GumboVector* vector) {
-  enlarge_vector_if_full(parser, vector);
+void gumbo_vector_add(void* element, GumboVector* vector) {
+  enlarge_vector_if_full(vector);
   assert(vector->data);
   assert(vector->length < vector->capacity);
   vector->data[vector->length++] = element;
 }
 
-void* gumbo_vector_pop(
-    struct GumboInternalParser* parser, GumboVector* vector) {
+void* gumbo_vector_pop(GumboVector* vector) {
   if (vector->length == 0) {
     return NULL;
   }
@@ -91,33 +74,38 @@ int gumbo_vector_index_of(GumboVector* vector, const void* element) {
   return -1;
 }
 
-void gumbo_vector_insert_at(struct GumboInternalParser* parser, void* element,
-    unsigned int index, GumboVector* vector) {
-  assert(index >= 0);
+void gumbo_vector_insert_at (
+  void* element,
+  unsigned int index,
+  GumboVector* vector
+) {
   assert(index <= vector->length);
-  enlarge_vector_if_full(parser, vector);
+  enlarge_vector_if_full(vector);
   ++vector->length;
-  memmove(&vector->data[index + 1], &vector->data[index],
-      sizeof(void*) * (vector->length - index - 1));
+  memmove (
+    &vector->data[index + 1],
+    &vector->data[index],
+    sizeof(void*) * (vector->length - index - 1)
+  );
   vector->data[index] = element;
 }
 
-void gumbo_vector_remove(
-    struct GumboInternalParser* parser, void* node, GumboVector* vector) {
+void gumbo_vector_remove(void* node, GumboVector* vector) {
   int index = gumbo_vector_index_of(vector, node);
   if (index == -1) {
     return;
   }
-  gumbo_vector_remove_at(parser, index, vector);
+  gumbo_vector_remove_at(index, vector);
 }
 
-void* gumbo_vector_remove_at(struct GumboInternalParser* parser,
-    unsigned int index, GumboVector* vector) {
-  assert(index >= 0);
+void* gumbo_vector_remove_at(unsigned int index, GumboVector* vector) {
   assert(index < vector->length);
   void* result = vector->data[index];
-  memmove(&vector->data[index], &vector->data[index + 1],
-      sizeof(void*) * (vector->length - index - 1));
+  memmove (
+    &vector->data[index],
+    &vector->data[index + 1],
+    sizeof(void*) * (vector->length - index - 1)
+  );
   --vector->length;
   return result;
 }

data/gumbo-parser/src/vector.h

@@ -1,19 +1,3 @@
-// Copyright 2010 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// Author: jdtang@google.com (Jonathan Tang)
-
 #ifndef GUMBO_VECTOR_H_
 #define GUMBO_VECTOR_H_
 
@@ -23,45 +7,39 @@
 extern "C" {
 #endif
 
-// Forward declaration since it's passed into some of the functions in this
-// header.
-struct GumboInternalParser;
-
 // Initializes a new GumboVector with the specified initial capacity.
-void gumbo_vector_init(struct GumboInternalParser* parser,
-    size_t initial_capacity, GumboVector* vector);
+void gumbo_vector_init(unsigned int initial_capacity, GumboVector* vector);
 
-// Frees the memory used by an GumboVector.  Does not free the contained
+// Frees the memory used by a GumboVector. Does not free the contained
 // pointers.
-void gumbo_vector_destroy(
-    struct GumboInternalParser* parser, GumboVector* vector);
+void gumbo_vector_destroy(GumboVector* vector);
 
-// Adds a new element to an GumboVector.
-void gumbo_vector_add(
-    struct GumboInternalParser* parser, void* element, GumboVector* vector);
+// Adds a new element to a GumboVector.
+void gumbo_vector_add(void* element, GumboVector* vector);
 
 // Removes and returns the element most recently added to the GumboVector.
-// Ownership is transferred to caller.  Capacity is unchanged.  If the vector is
+// Ownership is transferred to caller. Capacity is unchanged. If the vector is
 // empty, NULL is returned.
-void* gumbo_vector_pop(struct GumboInternalParser* parser, GumboVector* vector);
+void* gumbo_vector_pop(GumboVector* vector);
 
-// Inserts an element at a specific index.  This is potentially O(N) time, but
+// Inserts an element at a specific index. This is potentially O(N) time, but
 // is necessary for some of the spec's behavior.
-void gumbo_vector_insert_at(struct GumboInternalParser* parser, void* element,
-    unsigned int index, GumboVector* vector);
+void gumbo_vector_insert_at (
+  void* element,
+  unsigned int index,
+  GumboVector* vector
+);
 
 // Removes an element from the vector, or does nothing if the element is not in
 // the vector.
-void gumbo_vector_remove(
-    struct GumboInternalParser* parser, void* element, GumboVector* vector);
+void gumbo_vector_remove(void* element, GumboVector* vector);
 
-// Removes and returns an element at a specific index.  Note that this is
+// Removes and returns an element at a specific index. Note that this is
 // potentially O(N) time and should be used sparingly.
-void* gumbo_vector_remove_at(struct GumboInternalParser* parser,
-    unsigned int index, GumboVector* vector);
+void* gumbo_vector_remove_at(unsigned int index, GumboVector* vector);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // GUMBO_VECTOR_H_
+#endif // GUMBO_VECTOR_H_

data/lib/nokogumbo.rb

@@ -1,179 +1,15 @@
 require 'nokogiri'
-require 'nokogumboc'
+require 'nokogumbo/version'
+require 'nokogumbo/html5'
+require 'nokogumbo/xml/node.rb'
 
-module Nokogiri
-  # Parse an HTML document.  +string+ contains the document.  +string+
-  # may also be an IO-like object.  Returns a +Nokogiri::HTML::Document+.
-  def self.HTML5(*args)
-    Nokogiri::HTML5.parse(*args)
-  end
+require 'nokogumbo/nokogumbo'
 
-  module HTML5
-    # Parse an HTML document.  +string+ contains the document.  +string+
-    # may also be an IO-like object.  Returns a +Nokogiri::HTML::Document+.
-    def self.parse(string, options={})
-      if string.respond_to? :read
-        string = string.read
-      end
+module Nokogumbo
+  # The default maximum number of errors for parsing a document or a fragment.
+  DEFAULT_MAX_ERRORS = 0
 
-      # convert to UTF-8 (Ruby 1.9+) 
-      if string.respond_to?(:encoding) and string.encoding != Encoding::UTF_8
-        string = reencode(string)
-      end
-
-      Nokogumbo.parse(string.to_s, options[:max_parse_errors] || 0)
-    end
-
-    # Fetch and parse a HTML document from the web, following redirects,
-    # handling https, and determining the character encoding using HTML5
-    # rules.  +uri+ may be a +String+ or a +URI+.  +options+ contains
-    # http headers and special options.  Everything which is not a
-    # special option is considered a header.  Special options include:
-    #  * :follow_limit => number of redirects which are followed
-    #  * :basic_auth => [username, password]
-    def self.get(uri, options={})
-      headers = options.clone
-      headers = {:follow_limit => headers} if Numeric === headers # deprecated
-      limit=headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10
-
-      require 'net/http'
-      uri = URI(uri) unless URI === uri
-
-      http = Net::HTTP.new(uri.host, uri.port)
-
-      # TLS / SSL support
-      http.use_ssl = true if uri.scheme == 'https'
-
-      # Pass through Net::HTTP override values, which currently include:
-      #   :ca_file, :ca_path, :cert, :cert_store, :ciphers,
-      #   :close_on_empty_response, :continue_timeout, :key, :open_timeout,
-      #   :read_timeout, :ssl_timeout, :ssl_version, :use_ssl,
-      #   :verify_callback, :verify_depth, :verify_mode
-      options.each do |key, value|
-        http.send "#{key}=", headers.delete(key) if http.respond_to? "#{key}="
-      end
-
-      request = Net::HTTP::Get.new(uri.request_uri)
-
-      # basic authentication
-      auth = headers.delete(:basic_auth)
-      auth ||= [uri.user, uri.password] if uri.user and uri.password
-      request.basic_auth auth.first, auth.last if auth
-
-      # remaining options are treated as headers
-      headers.each {|key, value| request[key.to_s] = value.to_s}
-
-      response = http.request(request)
-
-      case response
-      when Net::HTTPSuccess
-        doc = parse(reencode(response.body, response['content-type']), options)
-        doc.instance_variable_set('@response', response)
-        doc.class.send(:attr_reader, :response)
-        doc
-      when Net::HTTPRedirection
-        response.value if limit <= 1
-        location = URI.join(uri, response['location'])
-        get(location, options.merge(:follow_limit => limit-1))
-      else
-        response.value
-      end
-    end
-
-    # while fragment is on the Gumbo TODO list, simulate it by doing
-    # a full document parse and ignoring the parent <html>, <head>, and <body>
-    # tags, and collecting up the children of each.
-    def self.fragment(*args)
-      doc = parse(*args)
-      fragment = Nokogiri::HTML::DocumentFragment.new(doc)
-
-      if doc.children.length != 1 or doc.children.first.name != 'html'
-        # no HTML?  Return document as is
-        fragment = doc
-      else
-        # examine children of HTML element
-        children = doc.children.first.children
-
-        # head is always first.  If present, take children but otherwise
-        # ignore the head element
-        if children.length > 0 and doc.children.first.name = 'head'
-          fragment << children.shift.children
-        end
-
-        # body may be next, or last.  If found, take children but otherwise
-        # ignore the body element.  Also take any remaining elements, taking
-        # care to preserve order.
-        if children.length > 0 and doc.children.first.name = 'body'
-          fragment << children.shift.children
-          fragment << children
-        elsif children.length > 0 and doc.children.last.name = 'body'
-          body = children.pop
-          fragment << children
-          fragment << body.children
-        else
-          fragment << children
-        end
-      end
-
-      # return result
-      fragment
-    end
-
-  private
-
-    # Charset sniffing is a complex and controversial topic that understandably
-    # isn't done _by default_ by the Ruby Net::HTTP library.  This being said,
-    # it is a very real problem for consumers of HTML as the default for HTML
-    # is iso-8859-1, most "good" producers use utf-8, and the Gumbo parser
-    # *only* supports utf-8.
-    #
-    # Accordingly, Nokogiri::HTML::Document.parse provides limited encoding
-    # detection.  Following this lead, Nokogiri::HTML5 attempts to do likewise,
-    # while attempting to more closely follow the HTML5 standard.
-    #
-    # http://bugs.ruby-lang.org/issues/2567
-    # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
-    #
-    def self.reencode(body, content_type=nil)
-      return body unless body.respond_to? :encoding
-
-      if body.encoding == Encoding::ASCII_8BIT
-        encoding = nil
-
-        # look for a Byte Order Mark (BOM)
-        if body[0..1] == "\xFE\xFF"
-          encoding = 'utf-16be'
-        elsif body[0..1] == "\xFF\xFE"
-          encoding = 'utf-16le'
-        elsif body[0..2] == "\xEF\xBB\xBF"
-          encoding = 'utf-8'
-        end
-
-        # look for a charset in a content-encoding header
-        if content_type
-          encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1]
-        end
-
-        # look for a charset in a meta tag in the first 1024 bytes
-        if not encoding
-          data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, '')
-          data.scan(/<meta.*?>/m).each do |meta|
-            encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1]
-          end
-        end
-
-        # if all else fails, default to the official default encoding for HTML
-        encoding ||= Encoding::ISO_8859_1
-
-        # change the encoding to match the detected or inferred encoding
-        begin
-          body.force_encoding(encoding)
-        rescue ArgumentError
-          body.force_encoding(Encoding::ISO_8859_1)
-        end
-      end
-
-      body.encode(Encoding::UTF_8)
-    end
-  end
+  # The default maximum depth of the DOM tree produced by parsing a document
+  # or fragment.
+  DEFAULT_MAX_TREE_DEPTH = 400
 end