nokogiri 1.6.1 → 1.6.2.rc1

data/ROADMAP.md

@@ -77,8 +77,7 @@
 ## Encoding
 
 We have a lot of issues open around encoding. How bad are things?
-Would it help if we deprecated support for Ruby 1.8.7? Somebody who
-knows encoding well should head this up.
+Somebody who knows encoding well should head this up.
 
 * Extract EncodingReader as a real object that can be injected
   https://groups.google.com/forum/#!msg/nokogiri-talk/arJeAtMqvkg/tGihB-iBRSAJ

data/Rakefile

@@ -1,5 +1,6 @@
 # -*- ruby -*-
 require 'rubygems'
+require 'shellwords'
 
 gem 'hoe'
 require 'hoe'
@@ -14,11 +15,92 @@ GENERATED_TOKENIZER = "lib/nokogiri/css/tokenizer.rb"
 CROSS_DIR           =  File.join(File.dirname(__FILE__), 'ports')
 
 def java?
-  !! (RUBY_PLATFORM =~ /java/)
+  /java/ === RUBY_PLATFORM
 end
 
 ENV['LANG'] = "en_US.UTF-8" # UBUNTU 10.04, Y U NO DEFAULT TO UTF-8?
 
+CrossRuby = Struct.new(:version, :host) {
+  def ver
+    @ver ||= version[/\A[^-]+/]
+  end
+
+  def api_ver_suffix
+    case ver
+    when /\A([2-9])\.([0-9])\./
+      "#{$1}#{$2}0"
+    when /\A1\.9\./
+      '191'
+    else
+      raise "unsupported version: #{ver}"
+    end
+  end
+
+  def platform
+    @platform ||=
+      case host
+      when /\Ax86_64-/
+        'x64-mingw32'
+      when /\Ai[3-6]86-/
+        'x86-mingw32'
+      else
+        raise "unsupported host: #{host}"
+      end
+  end
+
+  def tool(name)
+    (@binutils_prefix ||=
+      case platform
+      when 'x64-mingw32'
+        'x86_64-w64-mingw32-'
+      when 'x86-mingw32'
+        'i686-w64-mingw32-'
+      end) + name
+  end
+
+  def target
+    case platform
+    when 'x64-mingw32'
+      'pei-x86-64'
+    when 'x86-mingw32'
+      'pei-i386'
+    end
+  end
+
+  def libruby_dll
+    case platform
+    when 'x64-mingw32'
+      "x64-msvcrt-ruby#{api_ver_suffix}.dll"
+    when 'x86-mingw32'
+      "msvcrt-ruby#{api_ver_suffix}.dll"
+    end
+  end
+
+  def dlls
+    [
+      'kernel32.dll',
+      'msvcrt.dll',
+      'ws2_32.dll',
+      *(case
+        when ver >= '2.0.0'
+          'user32.dll'
+        end),
+      libruby_dll
+    ]
+  end
+}
+
+CROSS_RUBIES = File.read('.cross_rubies').lines.flat_map { |line|
+  case line
+  when /\A([^#]+):([^#]+)/
+    CrossRuby.new($1, $2)
+  else
+    []
+  end
+}
+
+ENV['RUBY_CC_VERSION'] ||= CROSS_RUBIES.map(&:ver).uniq.join(":")
+
 require 'tasks/nokogiri.org'
 
 HOE = Hoe.spec 'nokogiri' do
@@ -26,25 +108,31 @@ HOE = Hoe.spec 'nokogiri' do
   developer 'Mike Dalessio',   'mike.dalessio@gmail.com'
   developer 'Yoko Harada',     'yokolet@gmail.com'
   developer 'Tim Elliott',     'tle@holymonkey.com'
+  developer 'Akinori MUSHA',   'knu@idaemons.org'
 
   self.readme_file  = ['README',    ENV['HLANG'], 'rdoc'].compact.join('.')
   self.history_file = ['CHANGELOG', ENV['HLANG'], 'rdoc'].compact.join('.')
 
   self.extra_rdoc_files = FileList['*.rdoc','ext/nokogiri/*.c']
 
-  self.licenses = ['MIT']
 
   self.clean_globs += [
     'nokogiri.gemspec',
     'lib/nokogiri/nokogiri.{bundle,jar,rb,so}',
-    'lib/nokogiri/{1.9,2.0}',
+    'lib/nokogiri/[0-9].[0-9]',
+    'ports/*.installed',
+    'ports/{i[3-6]86,x86_64}-{w64-,}mingw32*',
+    'ports/libxml2',
+    'ports/libxslt',
     # GENERATED_PARSER,
     # GENERATED_TOKENIZER
   ]
 
-  self.extra_deps += [
-    ["mini_portile",    "~> 0.5.0"],
-  ]
+  unless java?
+    self.extra_deps += [
+      ["mini_portile",    "~> 0.5.2"],
+    ]
+  end
 
   self.extra_dev_deps += [
     ["hoe-bundler",     ">= 1.1"],
@@ -53,7 +141,7 @@ HOE = Hoe.spec 'nokogiri' do
     ["hoe-git",         ">= 1.4"],
     ["minitest",        "~> 2.2.2"],
     ["rake",            ">= 0.9"],
-    ["rake-compiler",   "~> 0.8.0"],
+    ["rake-compiler",   "~> 0.9.2"],
     ["racc",            ">= 1.4.6"],
     ["rexical",         ">= 1.0.5"]
   ]
@@ -69,6 +157,7 @@ HOE = Hoe.spec 'nokogiri' do
 
   self.testlib = :minitest
 end
+HOE.spec.licenses = ['MIT']
 
 # ----------------------------------------
 
@@ -77,7 +166,7 @@ def add_file_to_gem relative_path
   target_dir = File.dirname(target_path)
   mkdir_p target_dir unless File.directory?(target_dir)
   rm_f target_path
-  ln relative_path, target_path
+  safe_ln relative_path, target_path
   HOE.spec.files += [relative_path]
 end
 
@@ -92,6 +181,8 @@ if java?
     jruby_home = RbConfig::CONFIG['prefix']
     ext.ext_dir = 'ext/java'
     ext.lib_dir = 'lib/nokogiri'
+    ext.source_version = '1.6'
+    ext.target_version = '1.6'
     jars = ["#{jruby_home}/lib/jruby.jar"] + FileList['lib/*.jar']
     ext.classpath = jars.map { |x| File.expand_path x }.join ':'
   end
@@ -100,9 +191,11 @@ if java?
     add_file_to_gem 'lib/nokogiri/nokogiri.jar'
   end
 else
-  mingw_available = true
   begin
-    require 'tasks/cross_compile'
+    require 'rake/extensioncompiler'
+    # Ensure mingw compiler is installed
+    Rake::ExtensionCompiler.mingw_host
+    mingw_available = true
   rescue
     puts "WARNING: cross compilation not available: #{$!}"
     mingw_available = false
@@ -111,18 +204,24 @@ else
 
   HOE.spec.files.reject! { |f| f =~ %r{\.(java|jar)$} }
 
-  windows_p = RbConfig::CONFIG['target_os'] == 'mingw32' || RbConfig::CONFIG['target_os'] =~ /mswin/
-
-  unless windows_p || java?
-    task gem_build_path do
-      add_file_to_gem "dependencies.yml"
-
-      dependencies = YAML.load_file("dependencies.yml")
-      %w[libxml2 libxslt].each do |lib|
-        version = dependencies[lib]
-        archive = File.join("ports", "archives", "#{lib}-#{version}.tar.gz")
-        add_file_to_gem archive
-      end
+  dependencies = YAML.load_file("dependencies.yml")
+
+  task gem_build_path do
+    %w[libxml2 libxslt].each do |lib|
+      version = dependencies[lib]
+      archive = File.join("ports", "archives", "#{lib}-#{version}.tar.gz")
+      add_file_to_gem archive
+      patchesdir = File.join("ports", "patches", lib)
+      patches = `#{['git', 'ls-files', patchesdir].shelljoin}`.split("\n").grep(/\.patch\z/)
+      patches.each { |patch|
+        add_file_to_gem patch
+      }
+      (untracked = Dir[File.join(patchesdir, '*.patch')] - patches).empty? or
+        at_exit {
+          untracked.each { |patch|
+            puts "** WARNING: untracked patch file not added to gem: #{patch}"
+          }
+        }
     end
   end
 
@@ -131,12 +230,16 @@ else
     ext.config_options << ENV['EXTOPTS']
     if mingw_available
       ext.cross_compile  = true
-      ext.cross_platform = ["x86-mswin32-60", "x86-mingw32"]
-      ext.cross_config_options << "--with-xml2-include=#{File.join($recipes["libxml2"].path, 'include', 'libxml2')}"
-      ext.cross_config_options << "--with-xml2-lib=#{File.join($recipes["libxml2"].path, 'lib')}"
-      ext.cross_config_options << "--with-iconv-dir=#{$recipes["libiconv"].path}"
-      ext.cross_config_options << "--with-xslt-dir=#{$recipes["libxslt"].path}"
-      ext.cross_config_options << "--with-zlib-dir=#{CROSS_DIR}"
+      ext.cross_platform = CROSS_RUBIES.map(&:platform).uniq
+      ext.cross_config_options << "--enable-cross-build"
+      ext.cross_compiling do |spec|
+        libs = dependencies.map { |name, version| "#{name}-#{version}" }.join(', ')
+
+        spec.post_install_message = <<-EOS
+Nokogiri is built with the packaged libraries: #{libs}.
+        EOS
+        spec.files.reject! { |path| File.fnmatch?('ports/*', path) }
+      end
     end
   end
 end
@@ -193,9 +296,9 @@ task :java_debug do
 end
 
 if java?
-  task :test_18 => :test
-  task :test_19 do
-    ENV['JRUBY_OPTS'] = "--1.9"
+  task :test_19 => :test
+  task :test_20 do
+    ENV['JRUBY_OPTS'] = "--2.0"
     Rake::Task["test"].invoke
   end
 end
@@ -212,37 +315,37 @@ end
 
 # ----------------------------------------
 
-desc "build a windows gem without all the ceremony."
-task "gem:windows" => "gem" do
-  cross_rubies = ["1.9.3-p194", "2.0.0-p0"]
-  ruby_cc_version = cross_rubies.collect { |_| _.split("-").first }.join(":") # e.g., "1.8.7:1.9.2"
-  rake_compiler_config_path = "#{ENV['HOME']}/.rake-compiler/config.yml"
+def verify_dll(dll, cross_ruby)
+  dll_imports = cross_ruby.dlls
+  dump = `#{['env', 'LANG=C', cross_ruby.tool('objdump'), '-p', dll].shelljoin}`
+  raise "unexpected file format for generated dll #{dll}" unless /file format #{Regexp.quote(cross_ruby.target)}\s/ === dump
+  raise "export function Init_nokogiri not in dll #{dll}" unless /Table.*\sInit_nokogiri\s/mi === dump
+
+  # Verify that the expected DLL dependencies match the actual dependencies
+  # and that no further dependencies exist.
+  dll_imports_is = dump.scan(/DLL Name: (.*)$/).map(&:first).map(&:downcase).uniq
+  if dll_imports_is.sort != dll_imports.sort
+    raise "unexpected dll imports #{dll_imports_is.inspect} in #{dll}"
+  end
+  puts "#{dll}: Looks good!"
+end
 
+task :cross do
+  rake_compiler_config_path = File.expand_path("~/.rake-compiler/config.yml")
   unless File.exists? rake_compiler_config_path
-    raise "rake-compiler has not installed any cross rubies. try running 'env --unset=HOST rake-compiler cross-ruby VERSION=#{cross_rubies.first}'"
-  end
-  rake_compiler_config = YAML.load_file(rake_compiler_config_path)
-
-  # check that rake-compiler config contains the right patchlevels. see #279 for background,
-  # and http://blog.mmediasys.com/2011/01/22/rake-compiler-updated-list-of-supported-ruby-versions-for-cross-compilation/
-  # for more up-to-date docs.
-  cross_rubies.each do |version|
-    majmin, patchlevel = version.split("-")
-    rbconfig = "rbconfig-#{majmin}"
-    unless rake_compiler_config.key?(rbconfig) && rake_compiler_config[rbconfig] =~ /-#{patchlevel}/
-      raise "rake-compiler '#{rbconfig}' not #{patchlevel}. try running 'env --unset=HOST rake-compiler cross-ruby VERSION=#{version}'"
-    end
+    raise "rake-compiler has not installed any cross rubies. Try using rake-compiler-dev-box for building binary windows gems.'"
   end
 
-  # verify that --export-all is in the 1.9 rbconfig. see #279,#374,#375.
-  rbconfig_19 = rake_compiler_config["rbconfig-1.9.3"]
-  raise "rbconfig #{rbconfig_19} needs --export-all in its DLDFLAGS value" if File.read(rbconfig_19).split("\n").grep(/CONFIG\["DLDFLAGS"\].*--export-all/).empty?
-
-  rbconfig_20 = rake_compiler_config["rbconfig-2.0.0"]
-  raise "rbconfig #{rbconfig_20} needs --export-all in its DLDFLAGS value" if File.read(rbconfig_20).split("\n").grep(/CONFIG\["DLDFLAGS"\].*--export-all/).empty?
-
-  pkg_config_path = %w[libxslt libxml2].collect { |pkg| File.join($recipes[pkg].path, "lib/pkgconfig") }.join(":")
-  sh("env PKG_CONFIG_PATH=#{pkg_config_path} RUBY_CC_VERSION=#{ruby_cc_version} rake cross native gem") || raise("build failed!")
+  CROSS_RUBIES.each do |cross_ruby|
+    task "tmp/#{cross_ruby.platform}/nokogiri/#{cross_ruby.ver}/nokogiri.so" do |t|
+      # To reduce the gem file size strip mingw32 dlls before packaging
+      sh [cross_ruby.tool('strip'), '-S', t.name].shelljoin
+      verify_dll t.name, cross_ruby
+    end
+  end
 end
 
+desc "build a windows gem without all the ceremony."
+task "gem:windows" => %w[cross native gem]
+
 # vim: syntax=Ruby

data/build_all

@@ -2,14 +2,19 @@
 #
 #  script to build gems for all relevant platforms:
 #  - MRI et al (standard gem)
-#  - windows (x86-mingw32 and x86-mswin32-60)
+#  - windows (x86-mingw32 and x64-mingw32)
 #  - jruby
 #
 #  here's what I recommend for building all the gems:
 #
-#   1. set up a vagrant VM guest running ubuntu lucid 32-bit.
+#  set up a rake-compiler-dev-box as described here:
+#      https://github.com/tjschuck/rake-compiler-dev-box
+#  It is prepared with all the necessary build tools and environments.
+#
+#  or alternatively do:
+#   1. Setup a ubuntu 12.04+ machine
 #   2. install rvm, and install 1.9.3, 2.0.0 and jruby.
-#   3. `sudo apt-get install mingw32`
+#   3. `sudo apt-get install mingw32 gcc-mingw-w64-i686 gcc-mingw-w64-x86-64`
 #
 #  as you build, you may run into these problems:
 #
@@ -34,6 +39,8 @@
 #    RUBY_ENGINE (just comment it out)
 #
 
+DEFAULT_RUBY=2.0.0
+
 HOST=
 
 # Load RVM into a shell session *as a function*
@@ -45,16 +52,37 @@ else
     echo "ERROR: An RVM installation was not found.\n"
 fi
 
-function rvm_use {
+rvm_use () {
     current_ruby=$1
     rvm use "${1}@nokogiri" --create || rvm -v
 }
 
+rvm_do () {
+    local ver=$1
+    shift
+    rvm "${ver}@nokogiri" do "$@"
+}
+
+cross_rubies () {
+    grep -v '#' .cross_rubies
+}
+
+ruby_vers () {
+    cross_rubies | while read -r line; do
+        echo "${line%%[-:]*}"
+    done | uniq
+}
+
 set -o errexit
 
-# initialize
-rvm_use 1.8.7
-bundle install --quiet --local || bundle install
+for ver in `ruby_vers`; do
+    rvm_use $ver
+    gem install bundler --conservative
+    bundle install --quiet --local || bundle install
+done
+
+rvm_use $DEFAULT_RUBY
+
 rm -rf tmp pkg
 bundle exec rake clean
 
@@ -63,42 +91,39 @@ rm -rf gems
 mkdir -p gems
 
 # windows
-platform=$(uname -i)
-if [[ $platform =~ "64" ]] ; then
-    echo ""
-    echo "ERROR: You need to build the windows gem on a 32-bit machine!"
-    echo ""
-    exit 1
-fi
-rvm_use 1.8.7
-if [[ ! -a ${HOME}/.rake-compiler/ruby/ruby-1.8.7-p358/lib/ruby/1.8.7/x86_64-linux/rbconfig.rb ]] ; then
 
-    # if this fails around the purelib.rb thing, try varying the ruby
-    # used to run this script, and whether the HOST env var is set
-    # below.
+for verspec in `cross_rubies`; do
+    version="${verspec%%:*}"
+    ver="${version%%-*}"
+    host="${verspec#*:}"
+
+    case $ver in
+        1.9.3)
+            # Parallel make does not work in 1.9.3, as it has a
+            # missing dependency problem with main.o.
+            make="make"
+            ;;
+        *)
+            make="$MAKE"
+    esac
+
+    MAKE="$make" rvm_do $ver rake-compiler cross-ruby VERSION=$version HOST=$host debugflags="-g"
+done
 
-    bundle exec rake-compiler cross-ruby VERSION=1.8.7-p358 # HOST=i386-mingw32
-fi
-if [[ ! -a ${HOME}/.rake-compiler/ruby/ruby-1.9.3-p194/lib/ruby/1.9.1/x86_64-linux/rbconfig.rb ]] ; then
-    bundle exec rake-compiler cross-ruby VERSION=1.9.3-p194
-fi
-if [[ ! -a ${HOME}/.rake-compiler/ruby/ruby-2.0.0-p0/lib/ruby/2.0.0/x86_64-linux/rbconfig.rb ]] ; then
-    bundle exec rake-compiler cross-ruby VERSION=2.0.0-p0
-fi
-bundle exec rake cross
 bundle exec rake gem:windows
-cp -v pkg/nokogiri*x86-{mingw32,mswin32}*.gem gems
+cp -v pkg/nokogiri*{x86,x64}-mingw32*.gem gems
 
 # MRI
-rvm_use 1.8.7
+bundle exec rake clean
 bundle exec rake gem
 cp -v pkg/nokogiri*.gem gems # should only be one at this point in the script
 
 # jruby
 rvm_use jruby
+gem install bundler --conservative
 bundle install --quiet --local || bundle install
 bundle exec rake clean clobber
-rvm_use 1.8.7
+rvm_use $DEFAULT_RUBY
 bundle exec rake generate
 rvm_use jruby
 bundle exec rake gem