nokogiri 1.0.0 → 1.6.8.1

data/Rakefile

@@ -1,199 +1,289 @@
 # -*- ruby -*-
-
 require 'rubygems'
-require 'hoe'
+require 'shellwords'
 
-kind = Config::CONFIG['DLEXT']
-windows = RUBY_PLATFORM =~ /mswin/i ? true : false
+gem 'hoe'
+require 'hoe'
+Hoe.plugin :debugging
+Hoe.plugin :git
+Hoe.plugin :gemspec
+Hoe.plugin :bundler
 
-LIB_DIR = File.expand_path(File.join(File.dirname(__FILE__), 'lib'))
-$LOAD_PATH << LIB_DIR
+GENERATED_PARSER    = "lib/nokogiri/css/parser.rb"
+GENERATED_TOKENIZER = "lib/nokogiri/css/tokenizer.rb"
 
-GENERATED_PARSER = "lib/nokogiri/css/generated_parser.rb"
-GENERATED_TOKENIZER = "lib/nokogiri/css/generated_tokenizer.rb"
+def java?
+  /java/ === RUBY_PLATFORM
+end
 
-EXT = "ext/nokogiri/native.#{kind}"
+ENV['LANG'] = "en_US.UTF-8" # UBUNTU 10.04, Y U NO DEFAULT TO UTF-8?
 
-require 'nokogiri/version'
+CrossRuby = Struct.new(:version, :host) {
+  def ver
+    @ver ||= version[/\A[^-]+/]
+  end
 
-HOE = Hoe.new('nokogiri', Nokogiri::VERSION) do |p|
-  p.developer('Aaron Patterson', 'aaronp@rubyforge.org')
-  p.clean_globs = [
-    'ext/nokogiri/Makefile',
-    'ext/nokogiri/*.{o,so,bundle,a,log,dll}',
-    'ext/nokogiri/conftest.dSYM',
-    GENERATED_PARSER,
-    GENERATED_TOKENIZER,
-    'cross',
-  ]
-  p.spec_extras = { :extensions => ["Rakefile"] }
-end
+  def minor_ver
+    @minor_ver ||= ver[/\A\d\.\d(?=\.)/]
+  end
 
-namespace :gem do
-  namespace :dev do
-    task :spec do
-      File.open("#{HOE.name}.gemspec", 'w') do |f|
-        HOE.spec.version = "#{HOE.version}.#{Time.now.strftime("%Y%m%d%H%M%S")}"
-        f.write(HOE.spec.to_ruby)
-      end
+  def api_ver_suffix
+    case minor_ver
+    when nil
+      raise "unsupported version: #{ver}"
+    when '1.9'
+      '191'
+    else
+      minor_ver.delete('.') << '0'
     end
   end
 
-  namespace :win32 do
-    task :spec do
-      File.open("#{HOE.name}.gemspec", 'w') do |f|
-        HOE.spec.files += Dir['ext/nokogiri/**.{dll,so}']
-        if windows
-          HOE.spec.platform = Gem::Platform::CURRENT
-        else
-          HOE.spec.platform = 'x86-mswin32-60'
-        end
-        HOE.spec.extensions = []
-        f.write(HOE.spec.to_ruby)
+  def platform
+    @platform ||=
+      case host
+      when /\Ax86_64-/
+        'x64-mingw32'
+      when /\Ai[3-6]86-/
+        'x86-mingw32'
+      else
+        raise "unsupported host: #{host}"
       end
+  end
+
+  def tool(name)
+    (@binutils_prefix ||=
+      case platform
+      when 'x64-mingw32'
+        'x86_64-w64-mingw32-'
+      when 'x86-mingw32'
+        'i686-w64-mingw32-'
+      end) + name
+  end
+
+  def target
+    case platform
+    when 'x64-mingw32'
+      'pei-x86-64'
+    when 'x86-mingw32'
+      'pei-i386'
     end
   end
 
-  namespace :unix do
-    task :spec do
-      File.open("#{HOE.name}.gemspec", 'w') do |f|
-        f.write(HOE.spec.to_ruby)
-      end
+  def libruby_dll
+    case platform
+    when 'x64-mingw32'
+      "x64-msvcrt-ruby#{api_ver_suffix}.dll"
+    when 'x86-mingw32'
+      "msvcrt-ruby#{api_ver_suffix}.dll"
     end
   end
 
-  task :spec => ['gem:dev:spec']
-end
+  def dlls
+    [
+      'kernel32.dll',
+      'msvcrt.dll',
+      'ws2_32.dll',
+      *(case
+        when ver >= '2.0.0'
+          'user32.dll'
+        end),
+      libruby_dll
+    ]
+  end
+}
+
+CROSS_RUBIES = File.read('.cross_rubies').lines.flat_map { |line|
+  case line
+  when /\A([^#]+):([^#]+)/
+    CrossRuby.new($1, $2)
+  else
+    []
+  end
+}
 
-desc "Run code-coverage analysis"
-task :coverage do
-  rm_rf "coverage"
-  sh "rcov -x Library -I lib:test #{Dir[*HOE.test_globs].join(' ')}"
-end
+ENV['RUBY_CC_VERSION'] ||= CROSS_RUBIES.map(&:ver).uniq.join(":")
 
-file GENERATED_PARSER => "lib/nokogiri/css/parser.y" do |t|
-  sh "racc -o #{t.name} #{t.prerequisites.first}"
-end
+HOE = Hoe.spec 'nokogiri' do
+  developer 'Aaron Patterson', 'aaronp@rubyforge.org'
+  developer 'Mike Dalessio',   'mike.dalessio@gmail.com'
+  developer 'Yoko Harada',     'yokolet@gmail.com'
+  developer 'Tim Elliott',     'tle@holymonkey.com'
+  developer 'Akinori MUSHA',   'knu@idaemons.org'
 
-file GENERATED_TOKENIZER => "lib/nokogiri/css/tokenizer.rex" do |t|
-  sh "frex -i --independent -o #{t.name} #{t.prerequisites.first}"
-end
+  license "MIT"
+
+  self.readme_file  = "README.md"
+  self.history_file = ['CHANGELOG', ENV['HLANG'], 'rdoc'].compact.join('.')
+
+  self.extra_rdoc_files = FileList['*.rdoc','ext/nokogiri/*.c']
 
-task 'ext/nokogiri/Makefile' do
-  Dir.chdir('ext/nokogiri') do
-    ruby "extconf.rb"
+
+  self.clean_globs += [
+    'nokogiri.gemspec',
+    'lib/nokogiri/nokogiri.{bundle,jar,rb,so}',
+    'lib/nokogiri/[0-9].[0-9]'
+  ]
+  self.clean_globs += Dir.glob("ports/*").reject { |d| d =~ %r{/archives$} }
+
+  unless java?
+    self.extra_deps += [
+      ["mini_portile2",    "~> 2.1.0"], # keep version in sync with extconf.rb
+    ]
   end
-end
 
-task EXT => 'ext/nokogiri/Makefile' do
-  Dir.chdir('ext/nokogiri') do
-    sh 'make'
+  self.extra_dev_deps += [
+    ["hoe-bundler",        "~> 1.2.0"],
+    ["hoe-debugging",      "~> 1.2.1"],
+    ["hoe-gemspec",        "~> 1.0.0"],
+    ["hoe-git",            "~> 1.6.0"],
+    ["minitest",           "~> 5.8.4"],
+    ["rake",               "~> 10.5.0"],
+    ["rake-compiler",      "~> 0.9.2"],
+    ["rake-compiler-dock", "~> 0.5.1"],
+    ["racc",               "~> 1.4.14"],
+    ["rexical",            "~> 1.0.5"]
+  ]
+
+  if java?
+    self.spec_extras = { :platform => 'java' }
+  else
+    self.spec_extras = {
+      :extensions => ["ext/nokogiri/extconf.rb"],
+      :required_ruby_version => '>= 1.9.2'
+    }
   end
+
+  self.testlib = :minitest
 end
 
-task :build => [EXT, GENERATED_PARSER, GENERATED_TOKENIZER]
-
-namespace :build do
-  namespace :win32 do
-    file 'cross/bin/ruby.exe' => ['cross/ruby-1.8.6-p287'] do
-      Dir.chdir('cross/ruby-1.8.6-p287') do
-        str = ''
-        File.open('Makefile.in', 'rb') do |f|
-          f.each_line do |line|
-            if line =~ /^\s*ALT_SEPARATOR =/
-              str += "\t\t    " + 'ALT_SEPARATOR = "\\\\\"; \\'
-              str += "\n"
-            else
-              str += line
-            end
-          end
-        end
-        File.open('Makefile.in', 'wb') { |f| f.write str }
-        buildopts = if File.exists?('/usr/bin/i586-mingw32msvc-gcc')
-                      "--host=i586-mingw32msvc --target=i386-mingw32 --build=i686-linux"
-                    else
-                      "--host=i386-mingw32 --target=i386-mingw32"
-                    end
-        sh(<<-eocommand)
-          env ac_cv_func_getpgrp_void=no \
-            ac_cv_func_setpgrp_void=yes \
-            rb_cv_negative_time_t=no \
-            ac_cv_func_memcmp_working=yes \
-            rb_cv_binary_elf=no \
-            ./configure \
-            #{buildopts} \
-            --prefix=#{File.expand_path(File.join(Dir.pwd, '..'))}
-        eocommand
-        sh 'make'
-        sh 'make install'
-      end
-    end
+# ----------------------------------------
 
-    desc 'build cross compiled ruby'
-    task :ruby => 'cross/bin/ruby.exe'
+def add_file_to_gem relative_path
+  target_path = File.join gem_build_path, relative_path
+  target_dir = File.dirname(target_path)
+  mkdir_p target_dir unless File.directory?(target_dir)
+  rm_f target_path
+  safe_ln relative_path, target_path
+  HOE.spec.files += [relative_path]
+end
+
+def gem_build_path
+  File.join 'pkg', HOE.spec.full_name
+end
+
+if java?
+  # TODO: clean this section up.
+  require "rake/javaextensiontask"
+  Rake::JavaExtensionTask.new("nokogiri", HOE.spec) do |ext|
+    jruby_home = RbConfig::CONFIG['prefix']
+    ext.ext_dir = 'ext/java'
+    ext.lib_dir = 'lib/nokogiri'
+    ext.source_version = '1.6'
+    ext.target_version = '1.6'
+    jars = ["#{jruby_home}/lib/jruby.jar"] + FileList['lib/*.jar']
+    ext.classpath = jars.map { |x| File.expand_path x }.join ':'
+    ext.debug = true if ENV['JAVA_DEBUG']
   end
 
-  desc 'build nokogiri for win32'
-  task :win32 => [GENERATED_PARSER, GENERATED_TOKENIZER, 'build:externals', 'build:win32:ruby'] do
-    dash_i = File.expand_path(
-      File.join(File.dirname(__FILE__), 'cross/lib/ruby/1.8/i386-mingw32/')
-    )
-    Dir.chdir('ext/nokogiri') do
-      ruby " -I #{dash_i} extconf.rb"
-      sh 'make'
-    end
-    dlls = Dir[File.join(File.dirname(__FILE__), 'cross', '**/*.dll')]
-    dlls.each do |dll|
-      next if dll =~ /ruby/
-      cp dll, 'ext/nokogiri'
+  task gem_build_path => [:compile] do
+    add_file_to_gem 'lib/nokogiri/nokogiri.jar'
+  end
+else
+  begin
+    require 'rake/extensioncompiler'
+    # Ensure mingw compiler is installed
+    Rake::ExtensionCompiler.mingw_host
+    mingw_available = true
+  rescue
+    mingw_available = false
+  end
+  require "rake/extensiontask"
+
+  HOE.spec.files.reject! { |f| f =~ %r{\.(java|jar)$} }
+
+  dependencies = YAML.load_file("dependencies.yml")
+
+  task gem_build_path do
+    %w[libxml2 libxslt].each do |lib|
+      version = dependencies[lib]["version"]
+      archive = File.join("ports", "archives", "#{lib}-#{version}.tar.gz")
+      add_file_to_gem archive
+      patchesdir = File.join("patches", lib)
+      patches = `#{['git', 'ls-files', patchesdir].shelljoin}`.split("\n").grep(/\.patch\z/)
+      patches.each { |patch|
+        add_file_to_gem patch
+      }
+      (untracked = Dir[File.join(patchesdir, '*.patch')] - patches).empty? or
+        at_exit {
+          untracked.each { |patch|
+            puts "** WARNING: untracked patch file not added to gem: #{patch}"
+          }
+        }
     end
   end
 
-  libs = %w{
-    iconv-1.9.2.win32
-    zlib-1.2.3.win32
-    libxml2-2.7.2.win32
-    libxslt-1.1.24.win32
-  }
-
-  libs.each do |lib|
-    file "stash/#{lib}.zip" do |t|
-      puts "downloading #{lib}"
-      FileUtils.mkdir_p('stash')
-      Dir.chdir('stash') do 
-        url = "http://www.zlatkovic.com/pub/libxml/#{lib}.zip"
-        system("wget #{url} || curl -O #{url}")
-      end
-    end
-    file "cross/#{lib}" => ["stash/#{lib}.zip"] do |t|
-      puts "unzipping #{lib}.zip"
-      FileUtils.mkdir_p('cross')
-      Dir.chdir('cross') do
-        sh "unzip ../stash/#{lib}.zip"
-        sh "touch #{lib}"
+  Rake::ExtensionTask.new("nokogiri", HOE.spec) do |ext|
+    ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact)
+    ext.config_options << ENV['EXTOPTS']
+    if mingw_available
+      ext.cross_compile  = true
+      ext.cross_platform = CROSS_RUBIES.map(&:platform).uniq
+      ext.cross_config_options << "--enable-cross-build"
+      ext.cross_compiling do |spec|
+        libs = dependencies.map { |name, dep| "#{name}-#{dep["version"]}" }.join(', ')
+
+        spec.required_ruby_version = [
+          '>= 1.9.2',
+          "< #{CROSS_RUBIES.max_by(&:ver).minor_ver.succ}"
+        ]
+
+        spec.post_install_message = <<-EOS
+Nokogiri is built with the packaged libraries: #{libs}.
+        EOS
+        spec.files.reject! { |path| File.fnmatch?('ports/*', path) }
       end
     end
   end
+end
 
-  file "stash/ruby-1.8.6-p287.tar.gz" do |t|
-    puts "downloading ruby"
-    FileUtils.mkdir_p('stash')
-    Dir.chdir('stash') do 
-      url = ("ftp://ftp.ruby-lang.org/pub/ruby/1.8/ruby-1.8.6-p287.tar.gz")
-      system("wget #{url} || curl -O #{url}")
-    end
-  end
-  file 'cross/ruby-1.8.6-p287' => ["stash/ruby-1.8.6-p287.tar.gz"] do |t|
-    puts "unzipping ruby"
-    FileUtils.mkdir_p('cross')
-    Dir.chdir('cross') do
-      sh "tar zxvf ../stash/ruby-1.8.6-p287.tar.gz"
-    end
+# ----------------------------------------
+
+desc "Generate css/parser.rb and css/tokenizer.rex"
+task 'generate' => [GENERATED_PARSER, GENERATED_TOKENIZER]
+task 'gem:spec' => 'generate' if Rake::Task.task_defined?("gem:spec")
+
+# This is a big hack to make sure that the racc and rexical
+# dependencies in the Gemfile are constrainted to ruby platforms
+# (i.e. MRI and Rubinius). There's no way to do that through hoe,
+# and any solution will require changing hoe and hoe-bundler.
+old_gemfile_task = Rake::Task['bundler:gemfile'] rescue nil
+task 'bundler:gemfile' do
+  old_gemfile_task.invoke if old_gemfile_task
+
+  lines = File.open('Gemfile', 'r') { |f| f.readlines }.map do |line|
+    line =~ /racc|rexical/ ? "#{line.strip}, :platform => :ruby" : line
   end
+  File.open('Gemfile', 'w') { |f| lines.each { |line| f.puts line } }
+end
 
-  task :externals => libs.map { |x| "cross/#{x}" } + ['cross/ruby-1.8.6-p287']
+file GENERATED_PARSER => "lib/nokogiri/css/parser.y" do |t|
+  racc = RbConfig::CONFIG['target_os'] =~ /mswin32/ ? '' : `which racc`.strip
+  racc = "#{::RbConfig::CONFIG['bindir']}/racc" if racc.empty?
+  racc = %x{command -v racc}.strip if racc.empty?
+  sh "#{racc} -l -o #{t.name} #{t.prerequisites.first}"
 end
 
+file GENERATED_TOKENIZER => "lib/nokogiri/css/tokenizer.rex" do |t|
+  sh "rex --independent -o #{t.name} #{t.prerequisites.first}"
+end
+
+[:compile, :check_manifest].each do |task_name|
+  Rake::Task[task_name].prerequisites << GENERATED_PARSER
+  Rake::Task[task_name].prerequisites << GENERATED_TOKENIZER
+end
+
+# ----------------------------------------
+
 desc "set environment variables to build and/or test with debug options"
 task :debug do
   ENV['NOKOGIRI_DEBUG'] = "true"
@@ -201,64 +291,85 @@ task :debug do
   ENV['CFLAGS'] += " -DDEBUG"
 end
 
-def test_suite_cmdline
-  require 'find'
-  files = []
-  Find.find("test") do |f|
-    files << f if File.basename(f) =~ /.*test.*\.rb$/
-  end
-  cmdline = "ruby -w -I.:lib:ext:test -rtest/unit -e '%w[#{files.join(' ')}].each {|f| require f}'"
+require File.join File.dirname(__FILE__), 'tasks/test'
+
+task :java_debug do
+  ENV['JRUBY_OPTS'] = "#{ENV['JRUBY_OPTS']} --debug --dev"
+  ENV['JAVA_OPTS'] = '-Xdebug -Xrunjdwp:transport=dt_socket,address=8000,server=y,suspend=y' if ENV['JAVA_DEBUG']
 end
 
-namespace :test do
-  # partial-loads-ok and undef-value-errors necessary to ignore
-  # spurious (and eminently ignorable) warnings from the ruby
-  # interpreter
-  VALGRIND_BASIC_OPTS = "--num-callers=50 --error-limit=no --partial-loads-ok=yes --undef-value-errors=no"
-
-  desc "run test suite under valgrind with basic ruby options"
-  task :valgrind => :build do
-    cmdline = "valgrind #{VALGRIND_BASIC_OPTS} #{test_suite_cmdline}"
-    puts cmdline
-    system cmdline
+if java?
+  task :test_19 => :test
+  task :test_20 do
+    ENV['JRUBY_OPTS'] = "--2.0"
+    Rake::Task["test"].invoke
   end
+end
+
+Rake::Task[:test].prerequisites << :compile
+Rake::Task[:test].prerequisites << :java_debug
+Rake::Task[:test].prerequisites << :check_extra_deps unless java?
 
-  desc "run test suite under valgrind with memory-fill ruby options"
-  task :valgrind_mem => :build do
-    # fill malloced memory with "m" and freed memory with "f"
-    cmdline = "valgrind #{VALGRIND_BASIC_OPTS} --freelist-vol=100000000 --malloc-fill=6D --free-fill=66 #{test_suite_cmdline}"
-    puts cmdline
-    system cmdline
+if Hoe.plugins.include?(:debugging)
+  ['valgrind', 'valgrind:mem', 'valgrind:mem0'].each do |task_name|
+    Rake::Task["test:#{task_name}"].prerequisites << :compile
   end
+end
+
+task "test:libxml-ruby" do
+  ENV['TEST_NOKOGIRI_WITH_LIBXML_RUBY'] = "1"
+  warn "#{__FILE__}:#{__LINE__}: --- running tests with libxml-ruby loaded ---"
+  Rake::Task[:test].execute
+  ENV['TEST_NOKOGIRI_WITH_LIBXML_RUBY'] = nil
+end
+
+Rake::Task["test:libxml-ruby"].prerequisites << :compile
 
-  desc "run test suite under valgrind with memory-zero ruby options"
-  task :valgrind_mem0 => :build do
-    # fill malloced and freed memory with 0
-    cmdline = "valgrind #{VALGRIND_BASIC_OPTS} --freelist-vol=100000000 --malloc-fill=00 --free-fill=00 #{test_suite_cmdline}"
-    puts cmdline
-    system cmdline
+task "test:valgrind:libxml-ruby" do
+  ENV['TEST_NOKOGIRI_WITH_LIBXML_RUBY'] = "1"
+  warn "#{__FILE__}:#{__LINE__}: --- running tests with libxml-ruby loaded ---"
+  Rake::Task["test:valgrind"].execute
+  ENV['TEST_NOKOGIRI_WITH_LIBXML_RUBY'] = nil
+end
+
+Rake::Task["test:valgrind:libxml-ruby"].prerequisites << :compile
+
+# ----------------------------------------
+
+def verify_dll(dll, cross_ruby)
+  dll_imports = cross_ruby.dlls
+  dump = `#{['env', 'LANG=C', cross_ruby.tool('objdump'), '-p', dll].shelljoin}`
+  raise "unexpected file format for generated dll #{dll}" unless /file format #{Regexp.quote(cross_ruby.target)}\s/ === dump
+  raise "export function Init_nokogiri not in dll #{dll}" unless /Table.*\sInit_nokogiri\s/mi === dump
+
+  # Verify that the expected DLL dependencies match the actual dependencies
+  # and that no further dependencies exist.
+  dll_imports_is = dump.scan(/DLL Name: (.*)$/).map(&:first).map(&:downcase).uniq
+  if dll_imports_is.sort != dll_imports.sort
+    raise "unexpected dll imports #{dll_imports_is.inspect} in #{dll}"
   end
+  puts "#{dll}: Looks good!"
+end
 
-  desc "run test suite under gdb"
-  task :gdb => :build do
-    cmdline = "gdb --args #{test_suite_cmdline}"
-    puts cmdline
-    system cmdline
+task :cross do
+  rake_compiler_config_path = File.expand_path("~/.rake-compiler/config.yml")
+  unless File.exists? rake_compiler_config_path
+    raise "rake-compiler has not installed any cross rubies. Try using rake-compiler-dev-box for building binary windows gems.'"
   end
 
-  desc "run test suite with aggressive GC"
-  task :gc => :build do
-    ENV['NOKOGIRI_GC'] = "true"
-    Rake::Task["test"].invoke
+  CROSS_RUBIES.each do |cross_ruby|
+    task "tmp/#{cross_ruby.platform}/nokogiri/#{cross_ruby.ver}/nokogiri.so" do |t|
+      # To reduce the gem file size strip mingw32 dlls before packaging
+      sh [cross_ruby.tool('strip'), '-S', t.name].shelljoin
+      verify_dll t.name, cross_ruby
+    end
   end
 end
 
-
-# Only do this on unix, since we can't build on windows
-unless windows
-  Rake::Task[:test].prerequisites << :build
-  Rake::Task[:check_manifest].prerequisites << GENERATED_PARSER
-  Rake::Task[:check_manifest].prerequisites << GENERATED_TOKENIZER
+desc "build a windows gem without all the ceremony."
+task "gem:windows" do
+  require "rake_compiler_dock"
+  RakeCompilerDock.sh "bundle && rake cross native gem MAKE='nice make -j`nproc`' RUBY_CC_VERSION=#{ENV['RUBY_CC_VERSION']}"
 end
 
 # vim: syntax=Ruby

data/STANDARD_RESPONSES.md

@@ -0,0 +1,47 @@
+# Standard Responses to Requests
+
+These responses are needed often enough that I figured, let's just
+check them in for future reference and use.
+
+
+# Not enough information to help
+
+Hello!
+
+Thanks for asking this question! However, without more information,
+Team Nokogiri cannot reproduce your issue, and so we cannot offer much
+help.
+
+Please provide us with:
+
+* A self-contained script (one that we can run without modification,
+  and preferably without making external network connections).
+
+* Please note that you need to include the XML/HTML that you are
+  operating on.
+
+* The output of `nokogiri -v`, which will provide details about your
+  platform and versions of ruby, libxml2 and nokogiri.
+
+For more information about requesting help or reporting bugs, please
+take a look at http://bit.ly/nokohelp
+
+Thank you so much!
+
+
+# Not a bug
+
+Hello!
+
+Thanks for asking this question! Your request for assistance using
+Nokogiri will not go unanswered!
+
+However, Nokogiri's Github Issues is reserved for reporting bugs or
+submitting patches. If you ask your question on the mailing list, Team
+Nokogiri promises someone will provide you with an answer in a timely
+manner.
+
+If you'd like to read up on Team Nokogiri's rationale for this policy,
+please go to http://bit.ly/nokohelp.
+
+Thank you so much for understanding! And thank you for using Nokogiri.