superfeedr-nokogiri 1.4.0.20091116183308

data/Rakefile

@@ -0,0 +1,244 @@
+# -*- ruby -*-
+
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+
+windows = RUBY_PLATFORM =~ /(mswin|mingw)/i
+java    = RUBY_PLATFORM =~ /java/
+
+GENERATED_PARSER    = "lib/nokogiri/css/generated_parser.rb"
+GENERATED_TOKENIZER = "lib/nokogiri/css/generated_tokenizer.rb"
+
+# Make sure hoe-debugging is installed
+Hoe.plugin :debugging
+
+HOE = Hoe.spec 'nokogiri' do
+  developer('Aaron Patterson', 'aaronp@rubyforge.org')
+  developer('Mike Dalessio', 'mike.dalessio@gmail.com')
+  self.readme_file   = ['README', ENV['HLANG'], 'rdoc'].compact.join('.')
+  self.history_file  = ['CHANGELOG', ENV['HLANG'], 'rdoc'].compact.join('.')
+  self.extra_rdoc_files  = FileList['*.rdoc']
+  self.clean_globs = [
+    'lib/nokogiri/*.{o,so,bundle,a,log,dll}',
+    'lib/nokogiri/nokogiri.rb',
+    'lib/nokogiri/1.{8,9}',
+    GENERATED_PARSER,
+    GENERATED_TOKENIZER,
+    'cross',
+  ]
+
+  %w{ racc rexical rake-compiler }.each do |dep|
+    self.extra_dev_deps << [dep, '>= 0']
+  end
+
+  self.spec_extras = { :extensions => ["ext/nokogiri/extconf.rb"] }
+end
+
+task :ws_docs do
+  title = "#{HOE.name}-#{HOE.version} Documentation"
+
+  options = []
+  options << "--main=#{HOE.readme_file}"
+  options << '--format=activerecord'
+  options << '--threads=1'
+  options << "--title=#{title.inspect}"
+
+  options += HOE.spec.require_paths
+  options += HOE.spec.extra_rdoc_files
+  require 'rdoc/rdoc'
+  ENV['RAILS_ROOT'] ||= File.expand_path(File.join('..', 'nokogiri_ws'))
+  RDoc::RDoc.new.document options
+end
+
+unless java
+  gem 'rake-compiler', '>= 0.4.1'
+  require "rake/extensiontask"
+
+  RET = Rake::ExtensionTask.new("nokogiri", HOE.spec) do |ext|
+    ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact)
+
+    ext.config_options << ENV['EXTOPTS']
+    cross_dir = File.join(File.dirname(__FILE__), 'tmp', 'cross')
+    ext.cross_compile   = true
+    ext.cross_platform  = 'i386-mingw32'
+    ext.cross_config_options <<
+      "--with-iconv-dir=#{File.join(cross_dir, 'iconv')}"
+    ext.cross_config_options <<
+      "--with-xml2-dir=#{File.join(cross_dir, 'libxml2')}"
+    ext.cross_config_options <<
+      "--with-xslt-dir=#{File.join(cross_dir, 'libxslt')}"
+  end
+
+  file 'lib/nokogiri/nokogiri.rb' do
+    File.open("lib/#{HOE.name}/#{HOE.name}.rb", 'wb') do |f|
+      f.write <<-eoruby
+require "#{HOE.name}/\#{RUBY_VERSION.sub(/\\.\\d+$/, '')}/#{HOE.name}"
+      eoruby
+    end
+  end
+
+  namespace :cross do
+    task :file_list do
+      HOE.spec.platform = 'x86-mingw32'
+      HOE.spec.extensions = []
+      HOE.spec.files += Dir["lib/#{HOE.name}/#{HOE.name}.rb"]
+      HOE.spec.files += Dir["lib/#{HOE.name}/1.{8,9}/#{HOE.name}.so"]
+      HOE.spec.files += Dir["ext/nokogiri/*.dll"]
+    end
+  end
+
+  CLOBBER.include("lib/nokogiri/nokogiri.{so,dylib,rb,bundle}")
+  CLOBBER.include("lib/nokogiri/1.{8,9}")
+  CLOBBER.include("ext/nokogiri/*.dll")
+end
+
+namespace :gem do
+  namespace :dev do
+    task :spec => [ GENERATED_PARSER, GENERATED_TOKENIZER ] do
+      File.open("#{HOE.name}.gemspec", 'w') do |f|
+        HOE.spec.version = "#{HOE.version}.#{Time.now.strftime("%Y%m%d%H%M%S")}"
+        f.write(HOE.spec.to_ruby)
+      end
+    end
+  end
+
+  desc "Build a gem targetted for JRuby"
+  task :jruby => ['gem:jruby:spec'] do
+    system "gem build nokogiri.gemspec"
+    FileUtils.mkdir_p "pkg"
+    FileUtils.mv Dir.glob("nokogiri*-java.gem"), "pkg"
+  end
+
+  namespace :jruby do
+    task :spec => [GENERATED_PARSER, GENERATED_TOKENIZER] do
+      File.open("#{HOE.name}.gemspec", 'w') do |f|
+        HOE.spec.platform = 'java'
+        HOE.spec.files << GENERATED_PARSER
+        HOE.spec.files << GENERATED_TOKENIZER
+        HOE.spec.files += Dir["ext/nokogiri/*.dll"]
+        HOE.spec.extensions = []
+        f.write(HOE.spec.to_ruby)
+      end
+    end
+  end
+
+  task :spec => ['gem:dev:spec']
+end
+
+file GENERATED_PARSER => "lib/nokogiri/css/parser.y" do |t|
+  begin
+    racc = `which racc`.strip
+    racc = "#{::Config::CONFIG['bindir']}/racc" if racc.empty?
+    sh "#{racc} -l -o #{t.name} #{t.prerequisites.first}"
+  rescue
+    abort "need racc, sudo gem install racc"
+  end
+end
+
+file GENERATED_TOKENIZER => "lib/nokogiri/css/tokenizer.rex" do |t|
+  begin
+    sh "rex --independent -o #{t.name} #{t.prerequisites.first}"
+  rescue
+    abort "need rexical, sudo gem install rexical"
+  end
+end
+
+libs = %w{
+  iconv-1.9.2.win32
+  zlib-1.2.3.win32
+  libxml2-2.7.3.win32
+  libxslt-1.1.24.win32
+}
+
+lib_dlls = {
+  'iconv-1.9.2.win32'     => 'iconv.dll',
+  'zlib-1.2.3.win32'      => 'zlib1.dll',
+  'libxml2-2.7.3.win32'   => 'libxml2.dll',
+  'libxslt-1.1.24.win32'  => 'libxslt.dll',
+}
+
+libs.each do |lib|
+  libname = lib.split('-').first
+
+  file "tmp/stash/#{lib}.zip" do |t|
+    puts "downloading #{lib}"
+    FileUtils.mkdir_p('tmp/stash')
+    Dir.chdir('tmp/stash') do
+      url = "ftp://ftp.xmlsoft.org/libxml2/win32/#{lib}.zip"
+      system("wget #{url} || curl -O #{url}")
+    end
+  end
+
+  file "tmp/cross/#{libname}" => ["tmp/stash/#{lib}.zip"] do |t|
+    puts "unzipping #{lib}.zip"
+    FileUtils.mkdir_p('tmp/cross')
+    Dir.chdir('tmp/cross') do
+      sh "unzip ../stash/#{lib}.zip"
+      sh "cp #{lib}/bin/* #{lib}/lib" # put DLL in lib, so dirconfig works
+      sh "mv #{lib} #{lib.split('-').first}"
+      sh "touch #{lib.split('-').first}"
+    end
+  end
+
+  file "ext/nokogiri/#{lib_dlls[lib]}" => "tmp/cross/#{libname}" do |t|
+    Dir.chdir('tmp/cross') do
+      sh "cp #{libname}/bin/*.dll ../../ext/nokogiri/"
+    end
+  end
+
+  if Rake::Task.task_defined?(:cross)
+    Rake::Task[:cross].prerequisites << "ext/nokogiri/#{lib_dlls[lib]}"
+    Rake::Task[:cross].prerequisites << "lib/nokogiri/nokogiri.rb"
+    Rake::Task[:cross].prerequisites << "cross:file_list"
+  end
+  Rake::Task['gem:jruby:spec'].prerequisites << "ext/nokogiri/#{lib_dlls[lib]}"
+end
+
+require 'tasks/test'
+
+desc "set environment variables to build and/or test with debug options"
+task :debug do
+  ENV['NOKOGIRI_DEBUG'] = "true"
+  ENV['CFLAGS'] ||= ""
+  ENV['CFLAGS'] += " -DDEBUG"
+end
+
+# required_ruby_version
+
+# Only do this on unix, since we can't build on windows
+unless windows || java || ENV['NOKOGIRI_FFI']
+  [:compile, :check_manifest].each do |task_name|
+    Rake::Task[task_name].prerequisites << GENERATED_PARSER
+    Rake::Task[task_name].prerequisites << GENERATED_TOKENIZER
+  end
+
+  Rake::Task[:test].prerequisites << :compile
+  if Hoe.plugins.include?(:debugging)
+    ['valgrind', 'valgrind:mem', 'valgrind:mem0'].each do |task_name|
+      Rake::Task["test:#{task_name}"].prerequisites << :compile
+    end
+  end
+else
+  [:test, :check_manifest].each do |task_name|
+    if Rake::Task[task_name]
+      Rake::Task[task_name].prerequisites << GENERATED_PARSER
+      Rake::Task[task_name].prerequisites << GENERATED_TOKENIZER
+    end
+  end
+end
+
+namespace :install do
+  desc "Install rex and racc for development"
+  task :deps => %w(rexical racc)
+
+  task :racc do |t|
+    sh "sudo gem install racc"
+  end
+
+  task :rexical do
+    sh "sudo gem install rexical"
+  end
+end
+
+# vim: syntax=Ruby

data/bin/nokogiri

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'open-uri'
+require 'irb'
+require 'uri'
+require 'rubygems'
+require 'nokogiri'
+
+parse_class = Nokogiri
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Nokogiri: an HTML, XML, SAX, and Reader parser"
+  opts.define_head "Usage: nokogiri <uri|path> [options]"
+  opts.separator ""
+  opts.separator "Examples:"
+  opts.separator "  nokogiri http://www.ruby-lang.org/"
+  opts.separator "  nokogiri ./public/index.html"
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("--type [TYPE]", [:xml, :html]) do |v|
+    parse_class = {:xml => Nokogiri::XML, :html => Nokogiri::HTML}[v]
+  end
+
+  opts.on_tail("-?", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  opts.on_tail("-v", "--version", "Show version") do
+    require 'yaml'
+    puts Nokogiri::VERSION_INFO.to_yaml
+    exit
+  end
+end
+opts.parse!
+
+uri = ARGV.shift
+
+if uri.to_s.strip.empty?
+  puts opts
+  exit 1
+end
+
+@doc = parse_class.parse(open(uri).read)
+
+puts "Your document is stored in @doc..."
+IRB.start
+

data/ext/nokogiri/extconf.rb

@@ -0,0 +1,145 @@
+ENV['RC_ARCHS'] = '' if RUBY_PLATFORM =~ /darwin/
+
+# :stopdoc:
+
+require 'mkmf'
+
+ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+LIBDIR = Config::CONFIG['libdir']
+INCLUDEDIR = Config::CONFIG['includedir']
+
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'macruby'
+  $LIBRUBYARG_STATIC.gsub!(/-static/, '')
+end
+
+$CFLAGS << " #{ENV["CFLAGS"]}"
+if Config::CONFIG['target_os'] == 'mingw32'
+  $CFLAGS << " -DXP_WIN -DXP_WIN32 -DUSE_INCLUDED_VASPRINTF"
+elsif Config::CONFIG['target_os'] == 'solaris2'
+  $CFLAGS << " -DUSE_INCLUDED_VASPRINTF"
+else
+  $CFLAGS << " -g -DXP_UNIX"
+end
+
+$CFLAGS << " -O3 -Wall -Wcast-qual -Wwrite-strings -Wconversion -Wmissing-noreturn -Winline"
+
+HEADER_DIRS = [
+  # First search /opt/local for macports
+  '/opt/local/include',
+  '/opt/local/include/libxml2',
+
+  # Then search /usr/local for people that installed from source
+  '/usr/local/include',
+  '/usr/local/include/libxml2',
+
+  # Check the ruby install locations
+  INCLUDEDIR,
+  File.join(INCLUDEDIR, "libxml2"),
+
+  # Finally fall back to /usr
+  '/usr/include',
+  '/usr/include/libxml2',
+]
+
+LIB_DIRS = [
+  # First search /opt/local for macports
+  '/opt/local/lib',
+
+  # Then search /usr/local for people that installed from source
+  '/usr/local/lib',
+
+  # Check the ruby install locations
+  LIBDIR,
+
+  # Finally fall back to /usr
+  '/usr/lib',
+]
+
+iconv_dirs = dir_config('iconv', '/opt/local/include', '/opt/local/lib')
+unless ["", ""] == iconv_dirs
+  HEADER_DIRS.unshift iconv_dirs.first
+  LIB_DIRS.unshift iconv_dirs[1]
+end
+
+xml2_dirs = dir_config('xml2', '/opt/local/include/libxml2', '/opt/local/lib')
+unless ["", ""] == xml2_dirs
+  HEADER_DIRS.unshift xml2_dirs.first
+  LIB_DIRS.unshift xml2_dirs[1]
+end
+
+xslt_dirs = dir_config('xslt', '/opt/local/include/', '/opt/local/lib')
+unless ["", ""] == xslt_dirs
+  HEADER_DIRS.unshift xslt_dirs.first
+  LIB_DIRS.unshift xslt_dirs[1]
+end
+
+CUSTOM_DASH_I = []
+
+def nokogiri_find_header header_file, *paths
+  # mkmf in ruby 1.8.5 does not have the "checking_message" method
+  message = defined?(checking_message) ?
+    checking_message(header_file, paths) :
+    header_file
+
+  header = cpp_include header_file
+  checking_for message do
+    found = false
+    paths.each do |dir|
+      if File.exists?(File.join(dir, header_file))
+        opt = "-I#{dir}".quote
+        if try_cpp header, opt
+          unless CUSTOM_DASH_I.include? dir
+            $INCFLAGS = "#{opt} #{$INCFLAGS}"
+            CUSTOM_DASH_I << dir
+          end
+          found = dir
+          break
+        end
+      end
+    end
+    found ||= try_cpp(header)
+  end
+end
+
+unless nokogiri_find_header('iconv.h', *HEADER_DIRS)
+  abort "iconv is missing.  try 'port install iconv' or 'yum install iconv'"
+end
+
+unless nokogiri_find_header('libxml/parser.h', *HEADER_DIRS)
+  abort "libxml2 is missing.  try 'port install libxml2' or 'yum install libxml2-devel'"
+end
+
+unless nokogiri_find_header('libxslt/xslt.h', *HEADER_DIRS)
+  abort "libxslt is missing.  try 'port install libxslt' or 'yum install libxslt-devel'"
+end
+
+unless nokogiri_find_header('libexslt/exslt.h', *HEADER_DIRS)
+  abort "libxslt is missing.  try 'port install libxslt' or 'yum install libxslt-devel'"
+end
+
+unless find_library('xml2', 'xmlParseDoc', *LIB_DIRS)
+  abort "libxml2 is missing.  try 'port install libxml2' or 'yum install libxml2'"
+end
+
+unless find_library('xslt', 'xsltParseStylesheetDoc', *LIB_DIRS)
+  abort "libxslt is missing.  try 'port install libxslt' or 'yum install libxslt-devel'"
+end
+
+unless find_library('exslt', 'exsltFuncRegister', *LIB_DIRS)
+  abort "libxslt is missing.  try 'port install libxslt' or 'yum install libxslt-devel'"
+end
+
+have_func('xmlRelaxNGSetParserStructuredErrors')
+have_func('xmlRelaxNGSetParserStructuredErrors')
+have_func('xmlRelaxNGSetValidStructuredErrors')
+have_func('xmlSchemaSetValidStructuredErrors')
+have_func('xmlSchemaSetParserStructuredErrors')
+
+if ENV['CPUPROFILE']
+  unless find_library('profiler', 'ProfilerEnable', *LIB_DIRS)
+    abort "google performance tools are not installed"
+  end
+end
+
+create_makefile('nokogiri/nokogiri')
+# :startdoc:

data/ext/nokogiri/html_document.c

@@ -0,0 +1,145 @@
+#include <html_document.h>
+
+/*
+ * call-seq:
+ *  new
+ *
+ * Create a new document
+ */
+static VALUE new(int argc, VALUE *argv, VALUE klass)
+{
+  VALUE uri, external_id, rest, rb_doc;
+
+  rb_scan_args(argc, argv, "0*", &rest);
+  uri         = rb_ary_entry(rest, (long)0);
+  external_id = rb_ary_entry(rest, (long)1);
+
+  htmlDocPtr doc = htmlNewDoc(
+      RTEST(uri) ? (const xmlChar *)StringValuePtr(uri) : NULL,
+      RTEST(external_id) ? (const xmlChar *)StringValuePtr(external_id) : NULL
+  );
+  rb_doc = Nokogiri_wrap_xml_document(klass, doc);
+  rb_obj_call_init(rb_doc, argc, argv);
+  return rb_doc ;
+}
+
+/*
+ * call-seq:
+ *  read_io(io, url, encoding, options)
+ *
+ * Read the HTML document from +io+ with given +url+, +encoding+,
+ * and +options+.  See Nokogiri::HTML.parse
+ */
+static VALUE read_io( VALUE klass,
+                      VALUE io,
+                      VALUE url,
+                      VALUE encoding,
+                      VALUE options )
+{
+  const char * c_url    = NIL_P(url)      ? NULL : StringValuePtr(url);
+  const char * c_enc    = NIL_P(encoding) ? NULL : StringValuePtr(encoding);
+  VALUE error_list      = rb_ary_new();
+
+  xmlResetLastError();
+  xmlSetStructuredErrorFunc((void *)error_list, Nokogiri_error_array_pusher);
+
+  htmlDocPtr doc = htmlReadIO(
+      io_read_callback,
+      io_close_callback,
+      (void *)io,
+      c_url,
+      c_enc,
+      (int)NUM2INT(options)
+  );
+  xmlSetStructuredErrorFunc(NULL, NULL);
+
+  if(doc == NULL) {
+    xmlFreeDoc(doc);
+
+    xmlErrorPtr error = xmlGetLastError();
+    if(error)
+      rb_exc_raise(Nokogiri_wrap_xml_syntax_error((VALUE)NULL, error));
+    else
+      rb_raise(rb_eRuntimeError, "Could not parse document");
+
+    return Qnil;
+  }
+
+  VALUE document = Nokogiri_wrap_xml_document(klass, doc);
+  rb_iv_set(document, "@errors", error_list);
+  return document;
+}
+
+/*
+ * call-seq:
+ *  read_memory(string, url, encoding, options)
+ *
+ * Read the HTML document contained in +string+ with given +url+, +encoding+,
+ * and +options+.  See Nokogiri::HTML.parse
+ */
+static VALUE read_memory( VALUE klass,
+                          VALUE string,
+                          VALUE url,
+                          VALUE encoding,
+                          VALUE options )
+{
+  const char * c_buffer = StringValuePtr(string);
+  const char * c_url    = NIL_P(url)      ? NULL : StringValuePtr(url);
+  const char * c_enc    = NIL_P(encoding) ? NULL : StringValuePtr(encoding);
+  int len               = RSTRING_LEN(string);
+  VALUE error_list      = rb_ary_new();
+
+  xmlResetLastError();
+  xmlSetStructuredErrorFunc((void *)error_list, Nokogiri_error_array_pusher);
+
+  htmlDocPtr doc = htmlReadMemory(c_buffer, len, c_url, c_enc, (int)NUM2INT(options));
+  xmlSetStructuredErrorFunc(NULL, NULL);
+
+  if(doc == NULL) {
+    xmlFreeDoc(doc);
+
+    xmlErrorPtr error = xmlGetLastError();
+    if(error)
+      rb_exc_raise(Nokogiri_wrap_xml_syntax_error((VALUE)NULL, error));
+    else
+      rb_raise(rb_eRuntimeError, "Could not parse document");
+
+    return Qnil;
+  }
+
+  VALUE document = Nokogiri_wrap_xml_document(klass, doc);
+  rb_iv_set(document, "@errors", error_list);
+  return document;
+}
+
+/*
+ * call-seq:
+ *  type
+ *
+ * The type for this document
+ */
+static VALUE type(VALUE self)
+{
+  htmlDocPtr doc;
+  Data_Get_Struct(self, xmlDoc, doc);
+  return INT2NUM((long)doc->type);
+}
+
+VALUE cNokogiriHtmlDocument ;
+void init_html_document()
+{
+  VALUE nokogiri  = rb_define_module("Nokogiri");
+  VALUE html      = rb_define_module_under(nokogiri, "HTML");
+  VALUE xml       = rb_define_module_under(nokogiri, "XML");
+  VALUE node      = rb_define_class_under(xml, "Node", rb_cObject);
+  VALUE xml_doc   = rb_define_class_under(xml, "Document", node);
+  VALUE klass     = rb_define_class_under(html, "Document", xml_doc);
+
+  cNokogiriHtmlDocument = klass;
+
+  rb_define_singleton_method(klass, "read_memory", read_memory, 4);
+  rb_define_singleton_method(klass, "read_io", read_io, 4);
+  rb_define_singleton_method(klass, "new", new, -1);
+
+  rb_define_method(klass, "type", type, 0);
+}