nokogiri 1.8.5 → 1.9.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8a0b2422e8eb44142a8f8c1c0a0443b844db2a9d73c66bba59f239d800be9da
-  data.tar.gz: eee2e520dd02954386e3b3c83b02ff9fcb12339edcfc9aaf124835badd9b3239
+  metadata.gz: d658fc7d5957c3a7ebb52b4760fe6a3e1b65bacdacd70c1ed6e193aea0742d3d
+  data.tar.gz: 2078571efdde9bc3a78b97545f3edbe5953b6ae4b977604362fcc2d22b14469c
 SHA512:
-  metadata.gz: c967b49cdb174560fc8519d3e7e7252e4d3bd6e7fde2d13e3b052a9c050291e6ea375cb039fbb96ab9599516bf1dba33e3d606175c08a7db07e4c1abc1f16d4a
-  data.tar.gz: ff74b9432fddaff768ba2bda7547461c86ec65133b9c1818a178b75d4db603d1a200204a95e16c5b7fc563f39d2210446ddcf9d18a8eeb4939d597b957074b26
+  metadata.gz: 8683c1e51a8ede1daf265e9a33af1aba329b91489c75da8608d40f0c06d2823d833236f21f05914acd860b7722822d8f9a9f36f3120a907eb126be8fe5b984b9
+  data.tar.gz: 134e6617355b1bb6aac6729c7a0ca0ceeead11304eaf28dd383a8c90b1e20be83aa16246e7bb90abddaba20cf7f203baba49756f4931735814502542cb40ad3c

data/CHANGELOG.md

@@ -1,3 +1,37 @@
+# 1.9.0.rc1 / 2018-12-10
+
+## Security Notes
+
+* [JRuby] Upgrade Xerces dependency from 2.11.0 to 2.12.0 to address upstream vulnerability CVE-2012-0881 [#1831] (Thanks @grajagandev for reporting.)
+
+
+## Features
+
+* `XML::Attr#value=` allows HTML node attribute values to be set to either a blank string or an empty boolean attribute. [#1800]
+* Introduce `XML::Node#wrap` which does what `XML::NodeSet#wrap` has always done, but for a single node. [#1531] (Thanks, @ethirajsrinivasan!)
+* [MRI] Improve installation experience on macOS High Sierra (Darwin). [#1812, #1813] (Thanks, @gpakosz and @nurse!)
+* [MRI] Node#dup supports copying a node directly to a new document. See the method documentation for details.
+* [MRI] DocumentFragment#dup is now more memory-efficient, avoiding making unnecessary copies. [#1063]
+* [JRuby] NodeSet has been rewritten to improve performance! [#1795]
+
+
+## Bug fixes
+
+* `NodeSet#each` now returns `self` instead of zero. [#1822] (Thanks, @olehif!)
+* [MRI] Address a memory leak when using XML::Builder to create nodes with namespaces. [#1810]
+* [MRI] Address a memory leak when unparenting a DTD. [#1784] (Thanks, @stevecheckoway!)
+* [MRI] Decrease large memory usage when making nested XPath queries. [#1749]
+* [MRI] Use RbConfig::CONFIG instead of ::MAKEFILE_CONFIG to fix installations that use Makefile macros. [#1820] (Thanks, @nobu!)
+* [JRuby] Fix failing tests on JRuby 9.2.x
+* [JRuby] Fix default namespaces in nodes reparented into a different document [#1774]
+* [JRuby] Fix support for Java 9. [#1759] (Thanks, @Taywee!)
+
+
+## Dependencies
+
+* [MRI] Upgrade mini_portile2 dependency from `~> 2.3.0` to `~> 2.4.0`
+
+
 # 1.8.5 / 2018-10-04
 
 ## Security Notes

data/Gemfile

@@ -4,10 +4,10 @@
 
 source "https://rubygems.org/"
 
-gem "mini_portile2", "~>2.3.0"
+gem "mini_portile2", "~>2.4.0"
 
 gem "hoe-bundler", "~>1.2", :group => [:development, :test]
-gem "hoe-debugging", "~>1.4", :group => [:development, :test]
+gem "hoe-debugging", "~>2.0", :group => [:development, :test]
 gem "hoe-gemspec", "~>1.0", :group => [:development, :test]
 gem "hoe-git", "~>1.6", :group => [:development, :test]
 gem "minitest", "~>5.8.4", :group => [:development, :test]
@@ -17,7 +17,7 @@ gem "rake-compiler-dock", "~>0.6.2", :group => [:development, :test]
 gem "racc", "~>1.4.14", :group => [:development, :test], :platform => [:ruby, :mingw, :x64_mingw]
 gem "rexical", "~>1.0.5", :group => [:development, :test], :platform => [:ruby, :mingw, :x64_mingw]
 gem "concourse", "~>0.15", :group => [:development, :test]
-gem "rdoc", "~>4.0", :group => [:development, :test]
-gem "hoe", "~>3.16", :group => [:development, :test]
+gem "rdoc", ">=4.0", "<7", :group => [:development, :test]
+gem "hoe", "~>3.17", :group => [:development, :test]
 
 # vim: syntax=ruby

data/Manifest.txt

@@ -81,6 +81,8 @@ ext/java/nokogiri/internals/XalanDTMManagerPatch.java
 ext/java/nokogiri/internals/XmlDeclHandler.java
 ext/java/nokogiri/internals/XmlDomParserContext.java
 ext/java/nokogiri/internals/XmlSaxParser.java
+ext/java/nokogiri/internals/dom2dtm/DOM2DTM.java
+ext/java/nokogiri/internals/dom2dtm/DOM2DTMdefaultNamespaceDeclarationNode.java
 ext/java/nokogiri/internals/c14n/AttrCompare.java
 ext/java/nokogiri/internals/c14n/C14nHelper.java
 ext/java/nokogiri/internals/c14n/CanonicalFilter.java
@@ -108,7 +110,6 @@ ext/java/nokogiri/internals/c14n/NameSpaceSymbTable.java
 ext/java/nokogiri/internals/c14n/NodeFilter.java
 ext/java/nokogiri/internals/c14n/UtfHelpper.java
 ext/java/nokogiri/internals/c14n/XMLUtils.java
-ext/java/org/apache/xml/dtm/ref/dom2dtm/DOM2DTMExt.java
 ext/nokogiri/depend
 ext/nokogiri/extconf.rb
 ext/nokogiri/html_document.c

data/README.md

@@ -18,7 +18,6 @@ or CSS3 selectors.
 
 [![Concourse CI](https://ci.nokogiri.org/api/v1/teams/nokogiri-core/pipelines/nokogiri/jobs/ruby-2.4-system/badge)](https://ci.nokogiri.org/teams/nokogiri-core/pipelines/nokogiri?groups=master)
 [![Code Climate](https://codeclimate.com/github/sparklemotion/nokogiri.svg)](https://codeclimate.com/github/sparklemotion/nokogiri)
-[![Version Eye](https://www.versioneye.com/ruby/nokogiri/badge.png)](https://www.versioneye.com/ruby/nokogiri)
 [![Join the chat at https://gitter.im/sparklemotion/nokogiri](https://badges.gitter.im/sparklemotion/nokogiri.svg)](https://gitter.im/sparklemotion/nokogiri?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 

data/Rakefile

@@ -127,13 +127,13 @@ HOE = Hoe.spec 'nokogiri' do
 
   unless java?
     self.extra_deps += [
-      ["mini_portile2",    "~> 2.3.0"], # keep version in sync with extconf.rb
+      ["mini_portile2",    "~> 2.4.0"], # keep version in sync with extconf.rb
     ]
   end
 
   self.extra_dev_deps += [
     ["hoe-bundler",        "~> 1.2"],
-    ["hoe-debugging",      "~> 1.4"],
+    ["hoe-debugging",      "~> 2.0"],
     ["hoe-gemspec",        "~> 1.0"],
     ["hoe-git",            "~> 1.6"],
     ["minitest",           "~> 5.8.4"],
@@ -266,11 +266,21 @@ task 'bundler:gemfile' do
 end
 
 file GENERATED_PARSER => "lib/nokogiri/css/parser.y" do |t|
-  sh "racc -l -o #{t.name} #{t.prerequisites.first}"
+  if java?
+    warn "WARNING: #{GENERATED_PARSER} may be out of date:"
+    sh "ls -lt #{t.name} #{t.prerequisites.first}"
+  else
+    sh "racc -l -o #{t.name} #{t.prerequisites.first}"
+  end
 end
 
 file GENERATED_TOKENIZER => "lib/nokogiri/css/tokenizer.rex" do |t|
-  sh "rex --independent -o #{t.name} #{t.prerequisites.first}"
+  if java?
+    warn "WARNING: #{GENERATED_TOKENIZER} may be out of date:"
+    sh "ls -lt #{t.name} #{t.prerequisites.first}"
+  else
+    sh "rex --independent -o #{t.name} #{t.prerequisites.first}"
+  end
 end
 
 [:compile, :check_manifest].each do |task_name|

data/build_all

@@ -17,6 +17,10 @@ fi
 
 set -o errexit
 
+# check that we have the latest jruby
+rvm use jruby
+rvm use ruby
+
 rm -rf tmp pkg
 bundle exec rake clean clobber
 
@@ -37,7 +41,7 @@ cp -v pkg/nokogiri*.gem gems
 bundle exec rake clean clobber
 bundle exec rake generate
 
-rvm jruby
+rvm use jruby
 gem install bundler --conservative
 bundle install --quiet --local || bundle install
 bundle exec ruby -S rake gem

data/ext/nokogiri/extconf.rb

@@ -400,9 +400,11 @@ if openbsd? && !using_system_libraries?
   ENV['CFLAGS'] = "#{ENV['CFLAGS']} -I /usr/local/include"
 end
 
-RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC'] if ENV['CC']
+if ENV['CC']
+  RbConfig::CONFIG['CC'] = RbConfig::MAKEFILE_CONFIG['CC'] = ENV['CC']
+end
 # use same c compiler for libxml and libxslt
-ENV['CC'] = RbConfig::MAKEFILE_CONFIG['CC']
+ENV['CC'] = RbConfig::CONFIG['CC']
 
 $LIBS << " #{ENV["LIBS"]}"
 
@@ -432,7 +434,7 @@ if RUBY_PLATFORM =~ /mingw/i
   $CPPFLAGS << ' "-Idummypath"'
 end
 
-if RbConfig::MAKEFILE_CONFIG['CC'] =~ /gcc/
+if RbConfig::CONFIG['CC'] =~ /gcc/
   $CFLAGS << " -O3" unless $CFLAGS[/-O\d/]
   $CFLAGS << " -Wall -Wcast-qual -Wwrite-strings -Wmissing-noreturn -Winline"
 end
@@ -460,7 +462,7 @@ else
   # The gem version constraint in the Rakefile is not respected at install time.
   # Keep this version in sync with the one in the Rakefile !
   require 'rubygems'
-  gem 'mini_portile2', '~> 2.3.0'
+  gem 'mini_portile2', '~> 2.4.0'
   require 'mini_portile2'
   message "Using mini_portile version #{MiniPortile::VERSION}\n"
 
@@ -560,7 +562,8 @@ EOM
       *(libiconv_recipe ? "--with-iconv=#{libiconv_recipe.path}" : iconv_configure_flags),
       "--with-c14n",
       "--with-debug",
-      "--with-threads"
+      "--with-threads",
+      *(darwin? ? ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] : "")
     ]
   end
 
@@ -573,7 +576,8 @@ EOM
       "--without-python",
       "--without-crypto",
       "--with-debug",
-      "--with-libxml-prefix=#{sh_export_path(libxml2_recipe.path)}"
+      "--with-libxml-prefix=#{sh_export_path(libxml2_recipe.path)}",
+      *(darwin? ? ["RANLIB=/usr/bin/ranlib", "AR=/usr/bin/ar"] : "")
     ]
   end
 

data/ext/nokogiri/xml_attr.c

@@ -4,37 +4,40 @@
  * call-seq:
  *  value=(content)
  *
- * Set the value for this Attr to +content+
+ * Set the value for this Attr to +content+. Use `nil` to remove the value
+ * (e.g., a HTML boolean attribute).
  */
 static VALUE set_value(VALUE self, VALUE content)
 {
   xmlAttrPtr attr;
-  Data_Get_Struct(self, xmlAttr, attr);
+  xmlChar *value;
+  xmlNode *cur;
 
-  if (attr->children) { xmlFreeNodeList(attr->children); }
+  Data_Get_Struct(self, xmlAttr, attr);
 
+  if (attr->children) {
+    xmlFreeNodeList(attr->children);
+  }
   attr->children = attr->last = NULL;
 
-  if (content) {
-    xmlChar *buffer;
-    xmlNode *tmp;
-
-    /* Encode our content */
-    buffer = xmlEncodeEntitiesReentrant(attr->doc, (unsigned char *)StringValueCStr(content));
+  if (content == Qnil) {
+    return content;
+  }
 
-    attr->children = xmlStringGetNodeList(attr->doc, buffer);
-    attr->last = NULL;
-    tmp = attr->children;
+  value = xmlEncodeEntitiesReentrant(attr->doc, (unsigned char *)StringValueCStr(content));
+  if (xmlStrlen(value) == 0) {
+    attr->children = xmlNewDocText(attr->doc, value);
+  } else {
+    attr->children = xmlStringGetNodeList(attr->doc, value);
+  }
+  xmlFree(value);
 
-    /* Loop through the children */
-    for(tmp = attr->children; tmp; tmp = tmp->next) {
-      tmp->parent = (xmlNode *)attr;
-      tmp->doc = attr->doc;
-      if (tmp->next == NULL) { attr->last = tmp; }
+  for (cur = attr->children; cur; cur = cur->next) {
+    cur->parent = (xmlNode *)attr;
+    cur->doc = attr->doc;
+    if (cur->next == NULL) {
+      attr->last = cur;
     }
-
-    /* Free up memory */
-    xmlFree(buffer);
   }
 
   return content;
@@ -74,7 +77,9 @@ static VALUE new(int argc, VALUE *argv, VALUE klass)
   rb_node = Nokogiri_wrap_xml_node(klass, (xmlNodePtr)node);
   rb_obj_call_init(rb_node, argc, argv);
 
-  if (rb_block_given_p()) { rb_yield(rb_node); }
+  if (rb_block_given_p()) {
+    rb_yield(rb_node);
+  }
 
   return rb_node;
 }

data/ext/nokogiri/xml_document.c

@@ -7,7 +7,10 @@ static int dealloc_node_i(xmlNodePtr key, xmlNodePtr node, xmlDocPtr doc)
     xmlFreePropList((xmlAttrPtr)node);
     break;
   case XML_NAMESPACE_DECL:
-    xmlFree(node);
+    xmlFreeNs((xmlNsPtr)node);
+    break;
+  case XML_DTD_NODE:
+    xmlFreeDtd((xmlDtdPtr)node);
     break;
   default:
     if(node->parent == NULL) {

data/ext/nokogiri/xml_namespace.c

@@ -63,7 +63,7 @@ static int part_of_an_xpath_node_set_eh(xmlNsPtr node)
 
 VALUE Nokogiri_wrap_xml_namespace(xmlDocPtr doc, xmlNsPtr node)
 {
-  VALUE ns, document, node_cache;
+  VALUE ns = 0, document, node_cache;
 
   assert(doc->type == XML_DOCUMENT_NODE || doc->type == XML_HTML_DOCUMENT_NODE);
 
@@ -71,8 +71,6 @@ VALUE Nokogiri_wrap_xml_namespace(xmlDocPtr doc, xmlNsPtr node)
 
   if (doc->type == XML_DOCUMENT_FRAG_NODE) doc = doc->doc;
 
-  ns = Data_Wrap_Struct(cNokogiriXmlNamespace, 0, 0, node);
-
   if (DOC_RUBY_OBJECT_TEST(doc)) {
     document = DOC_RUBY_OBJECT(doc);
 
@@ -91,6 +89,8 @@ VALUE Nokogiri_wrap_xml_namespace(xmlDocPtr doc, xmlNsPtr node)
     }
 
     rb_iv_set(ns, "@document", document);
+  } else {
+    ns = Data_Wrap_Struct(cNokogiriXmlNamespace, 0, 0, node);
   }
 
   node->_private = (void *)ns;

data/ext/nokogiri/xml_namespace.h

@@ -7,8 +7,7 @@ void init_xml_namespace();
 
 extern VALUE cNokogiriXmlNamespace ;
 
-VALUE Nokogiri_wrap_xml_namespace(xmlDocPtr doc, xmlNsPtr node) ;
-VALUE Nokogiri_wrap_xml_namespace2(VALUE document, xmlNsPtr node) ;
+VALUE Nokogiri_wrap_xml_namespace(xmlDocPtr doc, xmlNsPtr node);
 
 #define NOKOGIRI_NAMESPACE_EH(node) ((node)->type == XML_NAMESPACE_DECL)
 

data/ext/nokogiri/xml_node.c

@@ -31,13 +31,13 @@ typedef xmlNodePtr (*pivot_reparentee_func)(xmlNodePtr, xmlNodePtr);
 static void relink_namespace(xmlNodePtr reparented)
 {
   xmlNodePtr child;
-  xmlNsPtr ns;
 
   if (reparented->type != XML_ATTRIBUTE_NODE &&
       reparented->type != XML_ELEMENT_NODE) { return; }
 
   if (reparented->ns == NULL || reparented->ns->prefix == NULL) {
-    xmlChar *name = 0, *prefix = 0;
+    xmlNsPtr ns = NULL;
+    xmlChar *name = NULL, *prefix = NULL;
 
     name = xmlSplitQName2(reparented->name, &prefix);
 
@@ -99,6 +99,25 @@ static void relink_namespace(xmlNodePtr reparented)
     }
   }
 
+  /*
+   *  Search our parents for an existing definition of current namespace,
+   *  because the definition it's pointing to may have just been removed nsDef.
+   *
+   *  And although that would technically probably be OK, I'd feel better if we
+   *  referred to a namespace that's still present in a node's nsDef somewhere
+   *  in the doc.
+   */
+  if (reparented->ns) {
+    xmlNsPtr ns = xmlSearchNs(reparented->doc, reparented, reparented->ns->prefix);
+    if (ns
+        && ns != reparented->ns
+        && xmlStrEqual(ns->prefix, reparented->ns->prefix)
+        && xmlStrEqual(ns->href, reparented->ns->href)
+       ) {
+      xmlSetNs(reparented, ns);
+    }
+  }
+
   /* Only walk all children if there actually is a namespace we need to */
   /* reparent. */
   if (NULL == reparented->ns) { return; }
@@ -282,7 +301,11 @@ ok:
     }
 
     if (original_ns_prefix_is_default && reparentee->ns != NULL && reparentee->ns->prefix != NULL) {
-      /* issue #391, where new node's prefix may become the string "default" */
+      /*
+       *  issue #391, where new node's prefix may become the string "default"
+       *  see libxml2 tree.c xmlNewReconciliedNs which implements this behavior.
+       */
+      xmlFree(reparentee->ns->prefix);
       reparentee->ns->prefix = NULL;
     }
   }
@@ -509,22 +532,39 @@ static VALUE internal_subset(VALUE self)
 /*
  * call-seq:
  *  dup
+ *  dup(depth)
+ *  dup(depth, new_parent_doc)
  *
- * Copy this node.  An optional depth may be passed in, but it defaults
- * to a deep copy.  0 is a shallow copy, 1 is a deep copy.
+ * Copy this node.
+ * An optional depth may be passed in. 0 is a shallow copy, 1 (the default) is a deep copy.
+ * An optional new_parent_doc may also be passed in, which will be the new
+ * node's parent document. Defaults to the current node's document.
+ * current document.
  */
 static VALUE duplicate_node(int argc, VALUE *argv, VALUE self)
 {
-  VALUE level;
+  VALUE r_level, r_new_parent_doc;
+  int level;
+  int n_args;
+  xmlDocPtr new_parent_doc;
   xmlNodePtr node, dup;
 
-  if(rb_scan_args(argc, argv, "01", &level) == 0) {
-    level = INT2NUM((long)1);
+  Data_Get_Struct(self, xmlNode, node);
+
+  n_args = rb_scan_args(argc, argv, "02", &r_level, &r_new_parent_doc);
+
+  if (n_args < 1) {
+    r_level = INT2NUM((long)1);
   }
+  level = (int)NUM2INT(r_level);
 
-  Data_Get_Struct(self, xmlNode, node);
+  if (n_args < 2) {
+    new_parent_doc = node->doc;
+  } else {
+    Data_Get_Struct(r_new_parent_doc, xmlDoc, new_parent_doc);
+  }
 
-  dup = xmlDocCopyNode(node, node->doc, (int)NUM2INT(level));
+  dup = xmlDocCopyNode(node, new_parent_doc, level);
   if(dup == NULL) { return Qnil; }
 
   nokogiri_root_node(dup);
@@ -1308,11 +1348,11 @@ static VALUE line(VALUE self)
  */
 static VALUE add_namespace_definition(VALUE self, VALUE prefix, VALUE href)
 {
-  xmlNodePtr node, namespacee;
+  xmlNodePtr node, namespace;
   xmlNsPtr ns;
 
   Data_Get_Struct(self, xmlNode, node);
-  namespacee = node ;
+  namespace = node ;
 
   ns = xmlSearchNs(
          node->doc,
@@ -1322,10 +1362,10 @@ static VALUE add_namespace_definition(VALUE self, VALUE prefix, VALUE href)
 
   if(!ns) {
     if (node->type != XML_ELEMENT_NODE) {
-      namespacee = node->parent;
+      namespace = node->parent;
     }
     ns = xmlNewNs(
-           namespacee,
+           namespace,
            (const xmlChar *)StringValueCStr(href),
            (const xmlChar *)(NIL_P(prefix) ? NULL : StringValueCStr(prefix))
          );
@@ -1333,7 +1373,7 @@ static VALUE add_namespace_definition(VALUE self, VALUE prefix, VALUE href)
 
   if (!ns) { return Qnil ; }
 
-  if(NIL_P(prefix) || node != namespacee) { xmlSetNs(node, ns); }
+  if(NIL_P(prefix) || node != namespace) { xmlSetNs(node, ns); }
 
   return Nokogiri_wrap_xml_namespace(node->doc, ns);
 }

data/lib/nokogiri/version.rb

@@ -1,6 +1,6 @@
 module Nokogiri
   # The version of Nokogiri you are using
-  VERSION = '1.8.5'
+  VERSION = '1.9.0.rc1'
 
   class VersionInfo # :nodoc:
     def jruby?

data/lib/nokogiri/xml/document_fragment.rb

@@ -25,6 +25,17 @@ module Nokogiri
         children.each { |child| child.parent = self }
       end
 
+      if Nokogiri.uses_libxml?
+        def dup
+          new_document = document.dup
+          new_fragment = XML::DocumentFragment.new(new_document)
+          children.each do |child|
+            child.dup(1, new_document).parent = new_fragment
+          end
+          new_fragment
+        end
+      end
+
       ###
       # return the name for DocumentFragment
       def name

data/lib/nokogiri/xml/node.rb

@@ -161,6 +161,18 @@ module Nokogiri
         end
       end
 
+
+      ###
+      # Add html around this node
+      #
+      # Returns self
+      def wrap(html)
+        new_parent = document.parse(html).first
+        add_next_sibling(new_parent)
+        new_parent.add_child(self)
+        self
+      end
+
       ###
       # Add +node_or_tags+ as a child of this Node.
       # +node_or_tags+ can be a Nokogiri::XML::Node, a ::DocumentFragment, a ::NodeSet, or a string containing markup.

data/lib/nokogiri/xml/node_set.rb

@@ -203,6 +203,7 @@ module Nokogiri
         0.upto(length - 1) do |x|
           yield self[x]
         end
+        self
       end
 
       ###
@@ -230,14 +231,9 @@ module Nokogiri
       end
 
       ###
-      # Wrap this NodeSet with +html+ or the results of the builder in +blk+
-      def wrap(html, &blk)
-        each do |j|
-          new_parent = document.parse(html).first
-          j.add_next_sibling(new_parent)
-          new_parent.add_child(j)
-        end
-        self
+      # Wrap this NodeSet with +html+
+      def wrap html
+        map { |node| node.wrap html }
       end
 
       ###

data/test/helper.rb

@@ -213,7 +213,7 @@ module Nokogiri
         .each do |name|
           define_method name do |*arguments|
             @items << [name, *arguments]
-            super *arguments
+            super(*arguments)
           end
         end
 

data/test/html/test_document.rb

@@ -707,6 +707,16 @@ eohtml
         assert_equal 'ISO-8859-1', html.encoding.name
       end
 
+      def test_leaking_dtd_nodes_after_internal_subset_removal
+        # see https://github.com/sparklemotion/nokogiri/issues/1784
+        #
+        # just checking that this doesn't raise a valgrind error. we
+        # don't otherwise have any test coverage for removing DTDs.
+        #
+        100.times do |i|
+          Nokogiri::HTML::Document.new.internal_subset.remove
+        end
+      end
     end
   end
 end

data/test/test_memory_leak.rb

@@ -13,6 +13,22 @@ class TestMemoryLeak < Nokogiri::TestCase
 EOF
   end
 
+  #
+  #  this suite is turned off unless the env var NOKOGIRI_GC is non-nil
+  #
+  #  to run any of these tests, do something like this on the commandline:
+  #
+  #    $ NOKOGIRI_GC=t ruby -Ilib:test \
+  #          test/test_memory_leak.rb \
+  #          -n /test_leaking_namespace_node_strings/
+  #
+  #  also see:
+  #
+  #    https://github.com/sparklemotion/nokogiri/issues/1603
+  #
+  #  which is an open issue to resurrect these tests and run them as
+  #  part of the CI pipeline.
+  #
   if ENV['NOKOGIRI_GC'] # turning these off by default for now
     def test_dont_hurt_em_why
       content = File.open("#{File.dirname(__FILE__)}/files/dont_hurt_em_why.xml").read
@@ -126,7 +142,7 @@ EOF
     end
 
     def test_in_context_parser_leak
-      loop do 
+      loop do
         doc = Nokogiri::XML::Document.new
         fragment1 = Nokogiri::XML::DocumentFragment.new(doc, '<foo/>')
         node = fragment1.children[0]
@@ -144,8 +160,62 @@ EOF
         doc.xpath('name(//node())')
       end
     end
+
+    def test_leaking_namespace_node_strings
+      # see https://github.com/sparklemotion/nokogiri/issues/1810 for memory leak report
+      ns = {'xmlns' => 'http://schemas.xmlsoap.org/soap/envelope/'}
+      20.times do
+        10_000.times do
+          Nokogiri::XML::Builder.new do |xml|
+            xml.send 'Envelope', ns do
+              xml.send 'Foobar', ns
+            end
+          end
+        end
+        puts MemInfo.rss
+      end
+    end
+
+    def test_leaking_namespace_node_strings_with_prefix
+      # see https://github.com/sparklemotion/nokogiri/issues/1810 for memory leak report
+      ns = {'xmlns:foo' => 'http://schemas.xmlsoap.org/soap/envelope/'}
+      20.times do
+        10_000.times do
+          Nokogiri::XML::Builder.new do |xml|
+            xml.send 'Envelope', ns do
+              xml.send 'Foobar', ns
+            end
+          end
+        end
+        puts MemInfo.rss
+      end
+    end
+
+    def test_leaking_dtd_nodes_after_internal_subset_removal
+      # see https://github.com/sparklemotion/nokogiri/issues/1784
+      100_000.times do |i|
+        doc = Nokogiri::HTML::Document.new
+        doc.internal_subset.remove
+        puts MemInfo.rss if (i % 1000 == 0)
+      end
+    end
   end # if NOKOGIRI_GC
 
+  module MemInfo
+    # from https://stackoverflow.com/questions/7220896/get-current-ruby-process-memory-usage
+    # this is only going to work on linux
+    PAGE_SIZE = `getconf PAGESIZE`.chomp.to_i rescue 4096
+    STATM_PATH = "/proc/#{Process.pid}/statm"
+    STATM_FOUND = File.exist?(STATM_PATH)
+
+    def self.rss
+      if STATM_FOUND
+        return (File.read(STATM_PATH).split(' ')[1].to_i * PAGE_SIZE) / 1024
+      end
+      return 0
+    end
+  end
+
   private
 
   def count_object_space_documents

data/test/xml/test_attr.rb

@@ -26,12 +26,34 @@ module Nokogiri
         assert_equal "Y&ent1;", street.value
       end
 
-      def test_value=
+      def test_set_value
         xml = Nokogiri::XML.parse(File.read(XML_FILE), XML_FILE)
         address = xml.xpath('//address')[3]
         street = address.attributes['street']
         street.value = "Y&ent1;"
         assert_equal "Y&ent1;", street.value
+        assert_includes %Q{ street="Y&ent1;"}, street.to_xml
+      end
+
+      def test_set_value_with_entity_string_in_html_file
+        html = Nokogiri::HTML("<html><body><div foo='asdf'>")
+        foo = html.at_css("div").attributes["foo"]
+        foo.value = "Y&ent1;"
+        assert_includes %Q{ foo="Y&amp;ent1;"}, foo.to_html
+      end
+
+      def test_set_value_with_blank_string_in_html_file
+        html = Nokogiri::HTML("<html><body><div foo='asdf'>")
+        foo = html.at_css("div").attributes["foo"]
+        foo.value = ""
+        assert_includes %Q{ foo=""}, foo.to_html
+      end
+
+      def test_set_value_of_boolean_attr_with_nil_in_html_file
+        html = Nokogiri::HTML("<html><body><div disabled='asdf'>")
+        disabled = html.at_css("div").attributes["disabled"]
+        disabled.value = nil
+        assert_includes %Q{ disabled}, disabled.to_html
       end
 
       def test_unlink # aliased as :remove

data/test/xml/test_builder.rb

@@ -331,6 +331,18 @@ module Nokogiri
         assert_nil doc.at_xpath("//*[local-name() = 'products']").namespace
       end
 
+      def test_builder_reuses_namespaces
+        # see https://github.com/sparklemotion/nokogiri/issues/1810 for memory leak report
+        builder = Nokogiri::XML::Builder.new
+        builder.send "envelope", {'xmlns' => 'http://schemas.xmlsoap.org/soap/envelope/'} do
+          builder.send "package", {'xmlns' => 'http://schemas.xmlsoap.org/soap/envelope/'}
+        end
+        envelope = builder.doc.at_css("envelope")
+        package = builder.doc.at_css("package")
+        assert_equal envelope.namespace, package.namespace
+        assert_equal envelope.namespace.object_id, package.namespace.object_id
+      end
+
       private
 
       def namespaces_defined_on(node)

data/test/xml/test_document_fragment.rb

@@ -263,6 +263,20 @@ EOS
         Nokogiri::XML::DocumentFragment.parse(input) # assert_nothing_raised
       end
 
+      def test_dup_creates_tree_with_identical_structure
+        original = Nokogiri::XML::DocumentFragment.parse("<div><p>hello</p></div>")
+        duplicate = original.dup
+        assert_equal original.to_html, duplicate.to_html
+      end
+
+      def test_dup_creates_mutable_tree
+        original = Nokogiri::XML::DocumentFragment.parse("<div><p>hello</p></div>")
+        duplicate = original.dup
+        duplicate.at_css("div").add_child("<b>hello there</b>")
+        assert_nil original.at_css("b")
+        assert_not_nil duplicate.at_css("b")
+      end
+
       if Nokogiri.uses_libxml?
         def test_for_libxml_in_context_fragment_parsing_bug_workaround
           10.times do

data/test/xml/test_node.rb

@@ -165,6 +165,44 @@ module Nokogiri
         assert_equal x.first.name, "span"
       end
 
+      def test_dup_is_deep_copy_by_default
+        doc = XML::Document.parse "<root><div><p>hello</p></div></root>"
+        div = doc.at_css "div"
+        node = div.dup
+        assert_equal 1, node.children.length
+        assert_equal "<p>hello</p>", node.children.first.to_html
+      end
+
+      def test_dup_deep_copy
+        doc = XML::Document.parse "<root><div><p>hello</p></div></root>"
+        div = doc.at_css "div"
+        node = div.dup(1)
+        assert_equal 1, node.children.length
+        assert_equal "<p>hello</p>", node.children.first.to_html
+      end
+
+      def test_dup_shallow_copy
+        doc = XML::Document.parse "<root><div><p>hello</p></div></root>"
+        div = doc.at_css "div"
+        node = div.dup(0)
+        assert_equal 0, node.children.length
+      end
+
+      if Nokogiri.uses_libxml?
+        def test_dup_to_another_document
+          doc1 = HTML::Document.parse "<root><div><p>hello</p></div></root>"
+          doc2 = HTML::Document.parse "<div></div>"
+
+          div = doc1.at_css "div"
+          duplicate_div = div.dup(1, doc2)
+
+          assert_not_nil doc1.at_css("div")
+          assert_equal doc2, duplicate_div.document
+          assert_equal 1, duplicate_div.children.length
+          assert_equal "<p>hello</p>", duplicate_div.children.first.to_html
+        end
+      end
+
       def test_subclass_dup
         subclass = Class.new(Nokogiri::XML::Node)
         node = subclass.new('foo', @xml).dup
@@ -415,12 +453,19 @@ module Nokogiri
         assert_equal ns, ns2
       end
 
-      def test_add_default_ns
+      def test_add_default_namespace
         node = @xml.at('address')
         node.add_namespace(nil, 'http://tenderlovemaking.com')
         assert_equal 'http://tenderlovemaking.com', node.namespaces['xmlns']
       end
 
+      def test_add_default_namespace_twice
+        node = @xml.at('address')
+        ns = node.add_namespace(nil, 'http://tenderlovemaking.com')
+        ns2 = node.add_namespace(nil, 'http://tenderlovemaking.com')
+        assert_equal ns.object_id, ns2.object_id
+      end
+
       def test_add_multiple_namespaces
         node = @xml.at('address')
 
@@ -1320,6 +1365,15 @@ eoxml
           node.add_next_sibling(Nokogiri::XML::Text.new('after', node.document))
         end
       end
+
+      def test_wrap
+        xml = '<root><thing><div class="title">important thing</div></thing></root>'
+        doc = Nokogiri::XML(xml)
+        thing = doc.at_css("thing")
+        thing.wrap("<wrapper/>")
+        assert_equal 'wrapper', thing.parent.name
+        assert_equal 'thing', doc.at_css("wrapper").children.first.name
+      end
     end
   end
 end

data/test/xml/test_node_reparenting.rb

@@ -204,7 +204,7 @@ module Nokogiri
               node = doc1.at_xpath("//value")
               node.remove
               doc2.add_child(node)
-              assert_match /<value>3<\/value>/, doc2.to_xml
+              assert_match(/<value>3<\/value>/, doc2.to_xml)
             end
           end
 
@@ -224,6 +224,59 @@ module Nokogiri
               node.add_child(child)
               assert @doc.at('//xmlns:second')
             end
+
+            describe "and a child node was added to a new doc with the a different namespace using the same prefix" do
+              before do
+                @doc = Nokogiri::XML %Q{<root xmlns:bar="http://tenderlovemaking.com/"><bar:first/></root>}
+                new_doc = Nokogiri::XML %Q{<newroot xmlns:bar="http://flavorjon.es/"/>}
+                assert node = @doc.at("//tenderlove:first", tenderlove: "http://tenderlovemaking.com/")
+                new_doc.root.add_child node
+                @doc = new_doc
+              end
+
+              it "serializes the doc with the proper default namespace" do
+                assert_match(/<bar:first\ xmlns:bar="http:\/\/tenderlovemaking.com\/"\/>/, @doc.to_xml)
+              end
+            end
+
+            describe "and a child node was added to a new doc with the same default namespaces" do
+              before do
+                new_doc = Nokogiri::XML %Q{<newroot xmlns="http://tenderlovemaking.com/"/>}
+                assert node = @doc.at("//tenderlove:first", tenderlove: "http://tenderlovemaking.com/")
+                new_doc.root.add_child node
+                @doc = new_doc
+              end
+
+              it "serializes the doc with the proper default namespace" do
+                assert_match(/<first>/, @doc.to_xml)
+              end
+            end
+
+            describe "and a child node was added to a new doc without any default namespaces" do
+              before do
+                new_doc = Nokogiri::XML "<newroot/>"
+                assert node = @doc.at("//tenderlove:first", tenderlove: "http://tenderlovemaking.com/")
+                new_doc.root.add_child node
+                @doc = new_doc
+              end
+
+              it "serializes the doc with the proper default namespace" do
+                assert_match(/<first xmlns=\"http:\/\/tenderlovemaking.com\/\">/, @doc.to_xml)
+              end
+            end
+
+            describe "and a child node became the root of a new doc" do
+              before do
+                new_doc = Nokogiri::XML::Document.new
+                assert node = @doc.at("//tenderlove:first", tenderlove: "http://tenderlovemaking.com/")
+                new_doc.root = node
+                @doc = new_doc
+              end
+
+              it "serializes the doc with the proper default namespace" do
+                assert_match(/<first xmlns=\"http:\/\/tenderlovemaking.com\/\">/, @doc.to_xml)
+              end
+            end
           end
 
           describe "given a parent node with a default and non-default namespace" do
@@ -249,7 +302,7 @@ module Nokogiri
                   @node.add_child(@child)
                   assert reparented = @doc.at('//bar:second', "bar" => "http://tenderlovemaking.com/")
                   assert reparented.namespace_definitions.empty?
-                  assert_equal @ns, reparented.namespace
+                  assert_equal @doc.root.namespace, reparented.namespace
                   assert_equal(
                     {
                       "xmlns"     => "http://tenderlovemaking.com/",
@@ -282,17 +335,20 @@ module Nokogiri
             end
 
             describe "and a child with a namespace matching the parent's non-default namespace" do
+              before do
+                @root_ns = @doc.root.namespace_definitions.detect { |x| x.prefix == "foo" }
+              end
+
               describe "set by #namespace=" do
                 before do
-                  @ns = @doc.root.namespace_definitions.detect { |x| x.prefix == "foo" }
-                  @child.namespace = @ns
+                  @child.namespace = @root_ns
                 end
 
                 it "inserts a node that inherits the matching parent namespace" do
                   @node.add_child(@child)
                   assert reparented = @doc.at('//bar:second', "bar" => "http://flavorjon.es/")
                   assert reparented.namespace_definitions.empty?
-                  assert_equal @ns, reparented.namespace
+                  assert_equal @root_ns, reparented.namespace
                   assert_equal(
                     {
                       "xmlns"     => "http://tenderlovemaking.com/",
@@ -312,7 +368,7 @@ module Nokogiri
                   @node.add_child(@child)
                   assert reparented = @doc.at('//bar:second', "bar" => "http://flavorjon.es/")
                   assert reparented.namespace_definitions.empty?
-                  assert_equal @ns, reparented.namespace
+                  assert_equal @root_ns, reparented.namespace
                   assert_equal(
                     {
                       "xmlns"     => "http://tenderlovemaking.com/",
@@ -550,6 +606,19 @@ module Nokogiri
           end
         end
 
+        describe "reparenting and preserving a reference to the original ns" do
+          it "should not cause illegal memory access" do
+            # this test will only cause a failure in valgrind. it
+            # drives out the reason why we can't call xmlFreeNs in
+            # relink_namespace and instead have to root the nsdef.
+            doc = Nokogiri::XML '<root xmlns="http://flavorjon.es/"><envelope /></root>'
+            elem = doc.create_element "package", {"xmlns" => "http://flavorjon.es/"}
+            ns = elem.namespace_definitions
+            doc.at_css("envelope").add_child elem
+            ns.inspect
+          end
+        end
+
         describe "reparenting into another document" do
           it "correctly sets default namespace of a reparented node" do
             # issue described in #391

data/test/xml/test_node_set.rb

@@ -511,6 +511,19 @@ module Nokogiri
         assert_equal 'employee', @xml.search("//wrapper").first.children[0].name
       end
 
+      def test_wrap_various_node_types
+        xml = '<root><foo>contents</foo></root>'
+        doc = Nokogiri::XML xml
+        nodes = doc.at_css("root").xpath(".//* | .//*/text()") # foo and "contents"
+        nodes.wrap("<wrapper/>")
+        wrappers = doc.css("wrapper")
+        assert_equal "root", wrappers.first.parent.name
+        assert_equal "foo", wrappers.first.children.first.name
+        assert_equal "foo", wrappers.last.parent.name
+        assert wrappers.last.children.first.text?
+        assert_equal "contents", wrappers.last.children.first.text
+      end
+
       def test_wrap_a_fragment
         frag = Nokogiri::XML::DocumentFragment.parse <<-EOXML
           <employees>
@@ -804,6 +817,12 @@ module Nokogiri
         assert_equal node_set.document, new_set.document
         assert new_set.respond_to?(:awesome!)
       end
+
+      def test_each_should_return_self
+        node_set1 = @xml.css("address")
+        node_set2 = node_set1.each {}
+        assert_equal node_set1, node_set2
+      end
     end
   end
 end

data/test/xml/test_text.rb

@@ -55,6 +55,15 @@ module Nokogiri
           node << Text.new('bar', Document.new)
         }
       end
+
+      def test_wrap
+        xml = '<root><thing><div class="title">important thing</div></thing></root>'
+        doc = Nokogiri::XML(xml)
+        text = doc.at_css("div").children.first
+        text.wrap("<wrapper/>")
+        assert_equal 'wrapper', text.parent.name
+        assert_equal 'wrapper', doc.at_css("div").children.first.name
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nokogiri
 version: !ruby/object:Gem::Version
-  version: 1.8.5
+  version: 1.9.0.rc1
 platform: ruby
 authors:
 - Aaron Patterson
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-05 00:00:00.000000000 Z
+date: 2018-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mini_portile2
@@ -22,14 +22,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: hoe-bundler
   requirement: !ruby/object:Gem::Requirement
@@ -50,14 +50,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: hoe-gemspec
   requirement: !ruby/object:Gem::Requirement
@@ -188,30 +188,36 @@ dependencies:
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.16'
+        version: '3.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.16'
+        version: '3.17'
 description: |-
   Nokogiri (鋸) is an HTML, XML, SAX, and Reader parser.  Among
   Nokogiri's many features is the ability to search documents via XPath
@@ -569,12 +575,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.8
 signing_key: 
 specification_version: 4
 summary: Nokogiri (鋸) is an HTML, XML, SAX, and Reader parser