nokogumbo 2.0.0.pre.alpha → 2.0.4

data/gumbo-parser/src/utf8.h

@@ -30,7 +30,9 @@ struct GumboInternalError;
 struct GumboInternalParser;
 
 // Unicode replacement char.
-extern const int kUtf8ReplacementChar;
+#define kUtf8ReplacementChar 0xFFFD
+#define kUtf8BomChar 0xFEFF
+#define kUtf8MaxChar 0x10FFFF
 
 typedef struct GumboInternalUtf8Iterator {
   // Points at the start of the code point most recently read into 'current'.
@@ -60,9 +62,23 @@ typedef struct GumboInternalUtf8Iterator {
   struct GumboInternalParser* _parser;
 } Utf8Iterator;
 
-// Returns true if this Unicode code point is in the list of characters
-// forbidden by the HTML5 spec, such as NUL bytes and undefined control chars.
-bool utf8_is_invalid_code_point(int c) CONST_FN;
+// Returns true if this Unicode code point is a surrogate.
+CONST_FN static inline bool utf8_is_surrogate(int c) {
+  return c >= 0xD800 && c <= 0xDFFF;
+}
+
+// Returns true if this Unicode code point is a noncharacter.
+CONST_FN static inline bool utf8_is_noncharacter(int c) {
+  return
+    (c >= 0xFDD0 && c <= 0xFDEF)
+    || ((c & 0xFFFF) == 0xFFFE)
+    || ((c & 0xFFFF) == 0xFFFF);
+}
+
+// Returns true if this Unicode code point is a control.
+CONST_FN static inline bool utf8_is_control(int c) {
+  return ((unsigned int)c < 0x1Fu) || (c >= 0x7F && c <= 0x9F);
+}
 
 // Initializes a new Utf8Iterator from the given byte buffer. The source does
 // not have to be NUL-terminated, but the length must be passed in explicitly.
@@ -77,20 +93,47 @@ void utf8iterator_init (
 void utf8iterator_next(Utf8Iterator* iter);
 
 // Returns the current code point as an integer.
-int utf8iterator_current(const Utf8Iterator* iter);
+static inline int utf8iterator_current(const Utf8Iterator* iter) {
+  return iter->_current;
+}
 
 // Retrieves and fills the output parameter with the current source position.
-void utf8iterator_get_position(
-    const Utf8Iterator* iter, GumboSourcePosition* output);
+static inline void utf8iterator_get_position (
+  const Utf8Iterator* iter,
+  GumboSourcePosition* output
+) {
+  *output = iter->_pos;
+}
+
+// Retrieves the marked position.
+static inline GumboSourcePosition utf8iterator_get_mark_position (
+  const Utf8Iterator* iter
+) {
+  return iter->_mark_pos;
+}
 
 // Retrieves a character pointer to the start of the current character.
-const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter);
+static inline const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter) {
+  return iter->_start;
+}
+
+// Retrieves the width of the current character.
+static inline size_t utf8iterator_get_width(const Utf8Iterator* iter) {
+  return iter->_width;
+}
 
 // Retrieves a character pointer to 1 past the end of the buffer. This is
 // necessary for certain state machines and string comparisons that would like
 // to look directly for ASCII text in the buffer without going through the
 // decoder.
-const char* utf8iterator_get_end_pointer(const Utf8Iterator* iter);
+static inline const char* utf8iterator_get_end_pointer(const Utf8Iterator* iter) {
+  return iter->_end;
+}
+
+// Retrieves a character pointer to the marked position.
+static inline const char* utf8iterator_get_mark_pointer(const Utf8Iterator* iter) {
+  return iter->_mark;
+}
 
 // If the upcoming text in the buffer matches the specified prefix (which has
 // length 'length'), consume it and return true. Otherwise, return false with
@@ -114,13 +157,6 @@ void utf8iterator_mark(Utf8Iterator* iter);
 // Returns the current input stream position to the mark.
 void utf8iterator_reset(Utf8Iterator* iter);
 
-// Sets the position and original text fields of an error to the value at the
-// mark.
-void utf8iterator_fill_error_at_mark (
-  Utf8Iterator* iter,
-  struct GumboInternalError* error
-);
-
 #ifdef __cplusplus
 }
 #endif

data/lib/nokogumbo.rb

@@ -1,11 +1,13 @@
 require 'nokogiri'
 require 'nokogumbo/version'
 require 'nokogumbo/html5'
-require 'nokogumbo/xml/node.rb'
 
 require 'nokogumbo/nokogumbo'
 
 module Nokogumbo
+  # The default maximum number of attributes per element.
+  DEFAULT_MAX_ATTRIBUTES = 400
+
   # The default maximum number of errors for parsing a document or a fragment.
   DEFAULT_MAX_ERRORS = 0
 

data/lib/nokogumbo/html5.rb

@@ -1,5 +1,6 @@
 require 'nokogumbo/html5/document'
 require 'nokogumbo/html5/document_fragment'
+require 'nokogumbo/html5/node'
 
 module Nokogiri
   # Parse an HTML 5 document. Convenience method for Nokogiri::HTML5::Document.parse
@@ -18,7 +19,7 @@ module Nokogiri
 
     # Parse an HTML 5 document. Convenience method for Nokogiri::HTML5::Document.parse
     def self.parse(string, url = nil, encoding = nil, **options, &block)
-      Document.parse(string, url, encoding, options, &block)
+      Document.parse(string, url, encoding, **options, &block)
     end
 
     # Parse a fragment from +string+. Convenience method for
@@ -91,19 +92,20 @@ module Nokogiri
         if encoding.nil?
           string = string.read
         else
-        string = string.read(encoding: encoding)
+          string = string.read(encoding: encoding)
         end
       else
         # Otherwise the string has the given encoding.
-        if encoding && string.respond_to?(:force_encoding)
+        string = string.to_s
+        if encoding
           string = string.dup
           string.force_encoding(encoding)
         end
       end
 
-      # convert to UTF-8 (Ruby 1.9+)
-      if string.respond_to?(:encoding) && string.encoding != Encoding::UTF_8
-        string = reencode(string.dup)
+      # convert to UTF-8
+      if string.encoding != Encoding::UTF_8
+        string = reencode(string)
       end
       string
     end
@@ -122,18 +124,17 @@ module Nokogiri
     # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding
     #
     def self.reencode(body, content_type=nil)
-      return body unless body.respond_to? :encoding
-
       if body.encoding == Encoding::ASCII_8BIT
         encoding = nil
 
         # look for a Byte Order Mark (BOM)
-        if body[0..1] == "\xFE\xFF"
-          encoding = 'utf-16be'
-        elsif body[0..1] == "\xFF\xFE"
-          encoding = 'utf-16le'
-        elsif body[0..2] == "\xEF\xBB\xBF"
-          encoding = 'utf-8'
+        initial_bytes = body[0..2].bytes
+        if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF]
+          encoding = Encoding::UTF_8
+        elsif initial_bytes[0..1] == [0xFE, 0xFF]
+          encoding = Encoding::UTF_16BE
+        elsif initial_bytes[0..1] == [0xFF, 0xFE]
+          encoding = Encoding::UTF_16LE
         end
 
         # look for a charset in a content-encoding header
@@ -153,6 +154,7 @@ module Nokogiri
         encoding ||= Encoding::ISO_8859_1
 
         # change the encoding to match the detected or inferred encoding
+        body = body.dup
         begin
           body.force_encoding(encoding)
         rescue ArgumentError
@@ -167,7 +169,7 @@ module Nokogiri
       case current_node.type
       when XML::Node::ELEMENT_NODE
         ns = current_node.namespace
-        ns_uri = ns.nil? ? nil : ns.uri
+        ns_uri = ns.nil? ? nil : ns.href
         # XXX(sfc): attach namespaces to all nodes, even html?
         if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE
           tagname = current_node.name

data/lib/nokogumbo/html5/document.rb

@@ -3,6 +3,7 @@ module Nokogiri
     class Document < Nokogiri::HTML::Document
       def self.parse(string_or_io, url = nil, encoding = nil, **options, &block)
         yield options if block_given?
+	string_or_io = '' unless string_or_io
 
         if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != 'ASCII-8BIT'
           encoding ||= string_or_io.encoding.name
@@ -11,24 +12,39 @@ module Nokogiri
         if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path)
           url ||= string_or_io.path
         end
+        unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str)
+          raise ArgumentError.new("not a string or IO object")
+        end
         do_parse(string_or_io, url, encoding, options)
       end
 
       def self.read_io(io, url = nil, encoding = nil, **options)
-        raise ArgumentError.new("io object doesn't respond to :read") unless io.respon_to?(:read)
+        raise ArgumentError.new("io object doesn't respond to :read") unless io.respond_to?(:read)
         do_parse(io, url, encoding, options)
       end
 
       def self.read_memory(string, url = nil, encoding = nil, **options)
-        do_parse(string.to_s, url, encoding, options)
+        raise ArgumentError.new("string object doesn't respond to :to_str") unless string.respond_to?(:to_str)
+        do_parse(string, url, encoding, options)
+      end
+
+      def fragment(tags = nil)
+        DocumentFragment.new(self, tags, self.root)
+      end
+
+      def to_xml(options = {}, &block)
+        # Bypass XML::Document#to_xml which doesn't add
+        # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does.
+        XML::Node.instance_method(:to_xml).bind(self).call(options, &block)
       end
 
       private
       def self.do_parse(string_or_io, url, encoding, options)
         string = HTML5.read_and_encode(string_or_io, encoding)
+        max_attributes = options[:max_attributes] || Nokogumbo::DEFAULT_MAX_ATTRIBUTES
         max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogumbo::DEFAULT_MAX_ERRORS
         max_depth = options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH
-        doc = Nokogumbo.parse(string.to_s, url, max_errors, max_depth)
+        doc = Nokogumbo.parse(string, url, max_attributes, max_errors, max_depth)
         doc.encoding = 'UTF-8'
         doc
       end

data/lib/nokogumbo/html5/document_fragment.rb

@@ -3,29 +3,20 @@ require 'nokogiri'
 module Nokogiri
   module HTML5
     class DocumentFragment < Nokogiri::HTML::DocumentFragment
+      attr_accessor :document
+      attr_accessor :errors
+
       # Create a document fragment.
       def initialize(doc, tags = nil, ctx = nil, options = {})
+        self.document = doc
+        self.errors = []
         return self unless tags
-        if ctx
-          raise Argument.new("Fragment parsing with context not supported")
-        else
-          tags = Nokogiri::HTML5.read_and_encode(tags, nil)
-
-          # Copied from Nokogiri's document_fragment.rb and labled "a horrible
-          # hack."
-          if tags.strip =~ /^<body/i
-            path = "/html/body"
-          else
-            path = "/html/body/node()"
-          end
-          # Add 2 for <html> and <body>.
-          max_depth = (options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH) + 2
-          options = options.dup
-          options[:max_tree_depth] = max_depth
-          temp_doc = HTML5.parse("<!DOCTYPE html><html><body>#{tags}", options)
-          temp_doc.xpath(path).each { |child| child.parent = self }
-        self.errors = temp_doc.errors
-        end
+
+        max_attributes = options[:max_attributes] || Nokogumbo::DEFAULT_MAX_ATTRIBUTES
+        max_errors = options[:max_errors] || Nokogumbo::DEFAULT_MAX_ERRORS
+        max_depth = options[:max_tree_depth] || Nokogumbo::DEFAULT_MAX_TREE_DEPTH
+        tags = Nokogiri::HTML5.read_and_encode(tags, nil)
+        Nokogumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth)
       end
 
       def serialize(options = {}, &block)
@@ -41,6 +32,31 @@ module Nokogiri
         doc.encoding = 'UTF-8'
         new(doc, tags, nil, options)
       end
+
+      def extract_params params # :nodoc:
+        handler = params.find do |param|
+          ![Hash, String, Symbol].include?(param.class)
+        end
+        params -= [handler] if handler
+
+        hashes = []
+        while Hash === params.last || params.last.nil?
+          hashes << params.pop
+          break if params.empty?
+        end
+        ns, binds = hashes.reverse
+
+        ns ||=
+          begin
+            ns = Hash.new
+            children.each { |child| ns.merge!(child.namespaces) }
+            ns
+          end
+
+        [params, handler, ns, binds]
+      end
+
     end
   end
 end
+# vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab:

data/lib/nokogumbo/{xml → html5}/node.rb

@@ -1,57 +1,72 @@
 require 'nokogiri'
 
 module Nokogiri
-  # Monkey patch
-  module XML
-    class Node
+  module HTML5
+    module Node
       # HTML elements can have attributes that contain colons.
       # Nokogiri::XML::Node#[]= treats names with colons as a prefixed QName
       # and tries to create an attribute in a namespace. This is especially
       # annoying with attribute names like xml:lang since libxml2 will
       # actually create the xml namespace if it doesn't exist already.
-      define_method(:add_child_node_and_reparent_attrs) do |node|
+      def add_child_node_and_reparent_attrs(node)
+        return super(node) unless document.is_a?(HTML5::Document)
+        # I'm not sure what this method is supposed to do. Reparenting
+        # namespaces is handled by libxml2, including child namespaces which
+        # this method wouldn't handle.
+        # https://github.com/sparklemotion/nokogiri/issues/1790
         add_child_node(node)
-        node.attribute_nodes.find_all { |a| a.namespace }.each do |attr|
-          attr.remove
-          node[attr.name] = attr.value
-        end
+        #node.attribute_nodes.find_all { |a| a.namespace }.each do |attr|
+        #  attr.remove
+        #  ns = attr.namespace
+        #  a["#{ns.prefix}:#{attr.name}"] = attr.value
+        #end
       end
 
       def inner_html(options = {})
+        return super(options) unless document.is_a?(HTML5::Document)
         result = options[:preserve_newline] && HTML5.prepend_newline?(self) ? "\n" : ""
         result << children.map { |child| child.to_html(options) }.join
         result
       end
 
       def write_to(io, *options)
+        return super(io, *options) unless document.is_a?(HTML5::Document)
         options = options.first.is_a?(Hash) ? options.shift : {}
         encoding = options[:encoding] || options[0]
         if Nokogiri.jruby?
           save_options = options[:save_with] || options[1]
           indent_times = options[:indent] || 0
         else
-          save_options = options[:save_with] || options[1] || SaveOptions::FORMAT
+          save_options = options[:save_with] || options[1] || XML::Node::SaveOptions::FORMAT
           indent_times = options[:indent] || 2
         end
         indent_string = (options[:indent_text] || ' ') * indent_times
 
-        config = SaveOptions.new(save_options.to_i)
+        config = XML::Node::SaveOptions.new(save_options.to_i)
         yield config if block_given?
 
         config_options = config.options
-        if (config_options & (SaveOptions::AS_XML | SaveOptions::AS_XHTML) != 0) || !document.is_a?(HTML5::Document)
+        if (config_options & (XML::Node::SaveOptions::AS_XML | XML::Node::SaveOptions::AS_XHTML) != 0)
           # Use Nokogiri's serializing code.
           native_write_to(io, encoding, indent_string, config_options)
         else
           # Serialize including the current node.
           encoding ||= document.encoding || Encoding::UTF_8
           internal_ops = {
-            trailing_nl: config_options & SaveOptions::FORMAT != 0,
             preserve_newline: options[:preserve_newline] || false
           }
-          HTML5.serialize_node_internal(self, io, encoding, options)
+          HTML5.serialize_node_internal(self, io, encoding, internal_ops)
         end
       end
+
+      def fragment(tags)
+        return super(tags) unless document.is_a?(HTML5::Document)
+        DocumentFragment.new(document, tags, self)
+      end
     end
+    # Monkey patch
+    XML::Node.prepend(HTML5::Node)
   end
 end
+
+# vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab:

data/lib/nokogumbo/version.rb

@@ -1,3 +1,3 @@
 module Nokogumbo
-  VERSION = "2.0.0-alpha"
+  VERSION = "2.0.4"
 end

metadata

@@ -1,30 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: nokogumbo
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre.alpha
+  version: 2.0.4
 platform: ruby
 authors:
 - Sam Ruby
 - Stephen Checkoway
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-31 00:00:00.000000000 Z
+date: 2020-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.8.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.8.4
 description: Nokogumbo allows a Ruby program to invoke the Gumbo HTML5 parser and
   access the result as a Nokogiri parsed document.
 email:
@@ -35,7 +41,6 @@ extensions:
 - ext/nokogumbo/extconf.rb
 extra_rdoc_files: []
 files:
-- CHANGELOG.md
 - LICENSE.txt
 - README.md
 - ext/nokogumbo/extconf.rb
@@ -63,6 +68,8 @@ files:
 - gumbo-parser/src/tag.c
 - gumbo-parser/src/tag_lookup.c
 - gumbo-parser/src/tag_lookup.h
+- gumbo-parser/src/token_buffer.c
+- gumbo-parser/src/token_buffer.h
 - gumbo-parser/src/token_type.h
 - gumbo-parser/src/tokenizer.c
 - gumbo-parser/src/tokenizer.h
@@ -77,8 +84,8 @@ files:
 - lib/nokogumbo/html5.rb
 - lib/nokogumbo/html5/document.rb
 - lib/nokogumbo/html5/document_fragment.rb
+- lib/nokogumbo/html5/node.rb
 - lib/nokogumbo/version.rb
-- lib/nokogumbo/xml/node.rb
 homepage: https://github.com/rubys/nokogumbo/#readme
 licenses:
 - Apache-2.0
@@ -87,7 +94,7 @@ metadata:
   changelog_uri: https://github.com/rubys/nokogumbo/blob/master/CHANGELOG.md
   homepage_uri: https://github.com/rubys/nokogumbo/#readme
   source_code_uri: https://github.com/rubys/nokogumbo
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -95,16 +102,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Nokogiri interface to the Gumbo HTML5 parser
 test_files: []