htmltools 1.10

data/INSTALL

@@ -0,0 +1,58 @@
+====================
+How to install:
+====================
+
+Configure:
+
+  ruby install.rb config
+
+Then, set up:
+
+  ruby install.rb setup
+
+Then, install (this may have to be done as root):
+
+  ruby install.rb install
+
+You can change paths, etc. using options to install.rb.
+
+install will automatically run tests and will not install if they failed.
+
+====================
+Usage of install.rb:
+====================
+  ruby install.rb <global option>
+  ruby install.rb [<global options>] <task> [<task options>]
+
+Global options:
+  -q,--quiet           suppress message outputs
+     --verbose         output messages verbosely
+  -h,--help            print this message
+  -v,--version         print version and quit
+  --copyright          print copyright and quit
+
+Tasks:
+  config      saves your configurations
+  show        shows current configuration
+  setup       compiles extention or else
+  install     installs files
+  clean       does `make clean' for each extention
+
+Options for config:
+  --prefix=path        path prefix of target environment [/usr/local]
+  --std-ruby=path      the directory for standard ruby libraries [$prefix/lib/ruby/1.6]
+  --site-ruby-common=path the directory for version-independent non-standard ruby libraries [$prefix/lib/ruby/site_ruby]
+  --site-ruby=path     the directory for non-standard ruby libraries [$prefix/lib/ruby/site_ruby/1.6]
+  --bin-dir=path       the directory for commands [$prefix/bin]
+  --rb-dir=path        the directory for ruby scripts [$site-ruby]
+  --so-dir=path        the directory for ruby extentions [$prefix/lib/ruby/site_ruby/1.6/i686-linux]
+  --data-dir=path      the directory for shared data [$prefix/share]
+  --ruby-path=path     path to set to #! line [/usr/local/bin/ruby]
+  --ruby-prog=name     the ruby program using for installation [/usr/local/bin/ruby]
+  --make-prog=name     the make program to compile ruby extentions [make]
+  --without-ext        does not compile/install ruby extentions [no]
+  --rbconfig=path      your rbconfig.rb to load [running ruby's]
+
+Options for install:
+  --no-harm            only display what to do if given [off]
+

data/README

@@ -0,0 +1,162 @@
+Version: 1.06
+12. September, 2003
+
+This is a Ruby library for building trees representing HTML structure.
+
+See the file INSTALL for installation instructions.
+
+Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>
+Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+
+License: Ruby's
+
+See http://rubyforge.org/projects/ruby-htmltools for the most recent version.
+
+
+
+This project includes SGML-parser, ported from Python by Takahiro Maebashi <maebashi@iij.ad.jp> (see: http://www.jin.gr.jp/~nahi/Ruby/html-parser/README.html)
+
+
+
+
+=============
+PREREQUISITES
+=============
+Ruby 1.8
+
+-------------
+Test::Unit
+-------------
+The tests run using Test::Unit. Test::Unit is part of the standard Ruby install
+as of 1.8
+
+-------------
+REXML
+-------------
+XPath support requires REXML. REXML is part of the standard Ruby install
+as of 1.8
+
+===========
+CHANGES
+===========
+------------------
+Changes from 1.09:
+------------------
+
+* Some minor bugfixes
+* SGMLParser.src_range makes it very easy to write applications which 
+  parse HTML files into components and manipulate the corresponding
+  source code _without_ altering it. (by Philip Dorrell)
+
+------------------
+Changes from 1.08:
+------------------
+
+* Fixed xpath script and added tests
+* Fixed bug #681 (xhtml)
+* Added GemSpec
+
+------------------
+Changes from 1.07:
+------------------
+
+* Fixed tc_xpath test_match_all after it was broken by upgrade of REXML.
+* Refactored utility code for printing node paths into rexml-nodepath.rb
+
+------------------
+Changes from 1.06:
+------------------
+
+* Included stuff that I had forgot to package into the tarball.
+
+------------------
+Changes from 1.05:
+------------------
+
+* Updated everything to work with Ruby 1.8.
+
+
+------------------
+Changes from 1.04:
+------------------
+
+* Made sure that unknown entities and characters are not discarded, in both
+  html/tree.rb and html/xmltree.rb
+
+* Added handling of DOCTYPE to html/xmltree.rb
+
+------------------
+Changes from 1.03:
+------------------
+
+* Added HTMLTree::XMLParser, which makes a REXML document from the given HTML.
+
+* Changed HTMLTree::Element::print_on() to write()
+
+* Made it so that a string or IO can be passed to HTMLTree::Element::dump()
+
+* Made it so that a string or IO can be passed to HTMLTree::Element::write()
+
+------------------
+Changes from 1.02:
+------------------
+
+* added XPath and XML conversion (needs REXML)
+
+* Wrapped all code in namespaces. The following class names have changed:
+
+  -- in html/element.rb
+  HTMLDocument => HTMLTree::Document
+  HTMLElement => HTMLTree::Element
+  HTMLData => HTMLTree::Data
+  HTMLComment => HTMLTree::Comment
+  HTMLSpecial => HTMLTree::Special
+
+  -- in html/tags.rb
+  HTMLTag => HTML::Tag
+  HTMLBlockTag => HTML::BlockTag
+  HTMLInlineTag => HTML::InlineTag
+  HTMLBlockOrInlineTag => HTML::BlockOrInlineTag
+  HTMLEmptyTag => HTML::EmptyTag
+
+  -- in html/tree.rb
+  HTMLTreeParser => HTMLTree::Parser
+
+  -- in html/stparser.rb
+  StackingParser => HTML::StackingParser
+
+* added HTMLTree::Element.root() 
+
+------------------
+Changes from 1.01:
+------------------
+
+* documented change to sgml-parser.
+
+* added bin/ebaySearch.rb example
+
+------------------
+Changes from 1.0:
+------------------
+
+* attributes now maintain their order. Though this probably isn't
+  strictly necessary under HTML, it may make it easier to compare
+  document versions.
+
+* the generated tree now has a top-level node for the document itself,
+  so the DTD can be stored. THIS WILL REQUIRE CODE CHANGES if you have
+  code that assumes that the root node is always <html>. To find the
+  <html> node, you can use the new methods HTMLTreeParser#html() or
+  HTMLDocument#html_node():
+
+     html = parser.html()
+
+  Or, querying the tree:
+
+     html = parser.tree.html_node()
+
+* comments are stored in the tree
+
+* added HTMLElement#print_on() to print a (sub)tree to an IO stream
+
+vim: ts=2 sw=2 et

data/demo/degolive.rb

@@ -0,0 +1,89 @@
+#!c:/ruby-1.8/bin/ruby
+# This removes GoLive tags and attributes.
+#
+# usage:
+#   ruby degolive.rb \<file> > <file>
+#
+# or (changes the file in place and saves the originals as .bak):
+#   ruby -i.bak degolive.rb files
+#
+# Copyright:: Copyright (C) 2002 Ned Konz
+# License::   Ruby License
+# CVS ID::    $Id: degolive.rb,v 1.1 2003/09/12 18:41:04 jhannes Exp $
+#
+require 'html/tags'
+require 'html/stparser'
+
+# Add nasty GoLive tags so we can remove them
+#                (name,      is_block, is_inline, is_empty, can_omit)
+HTML::Tag.add_tag('CSACTIONS',    true,   false,  false,  false)
+HTML::Tag.add_tag('CSACTION',     false,  true,   true,   false)
+HTML::Tag.add_tag('CSSCRIPTDICT', true,   false,  false,  false)
+HTML::Tag.add_tag('CSACTIONDICT', true,   false,  false,  false)
+
+class GoLiveRemover < HTML::StackingParser
+  # return true if we are in the scope of a bad tag
+  def ignoring(tag=nil)
+    (tag and tag =~ /^cs[as]/i) or
+    last_tag =~ /^cs[as]/i or
+    parent_tag =~ /^cs[as]/i
+  end
+
+  def printTag(tag, isStart=false, attrs=nil)
+    print(isStart ? "<" : "</")
+    print tag
+    if attrs
+      attrs.each { |a|
+        # Also need to remove 'csclick="..."'
+        # and on.*="CSAction(..." attribs
+        print " #{a[0]}=\"#{a[1]}\"" \
+          unless a[0] == "csclick" or (a[1] || '') =~ /^CSAction\(/
+      }
+    end
+    print(">")
+  end
+
+  def handle_start_tag(tag, attrs)
+    printTag(tag, true, attrs) unless ignoring(tag)
+  end
+
+  def handle_empty_tag(tag, attrs)
+    printTag(tag, true, attrs) unless ignoring(tag)
+  end
+
+  def handle_end_tag(tag)
+    printTag(tag, false) unless ignoring(tag)
+  end
+
+  def handle_missing_end_tag(tag)
+    warn("warning: inserting missing end tag </#{tag}>\n")
+    print("</#{tag}><!-- inserted -->")
+  end
+
+  def handle_data(data)
+    print(data)  unless ignoring
+  end
+
+  def handle_script(data)
+    print(data) unless ignoring
+  end
+
+  def handle_unknown_character(name)
+    print("&\##{name};") unless ignoring
+  end
+
+  def handle_unknown_entity(name)
+    print("&#{name};") unless ignoring
+  end
+
+  def handle_comment(data)
+    print(data) unless ignoring
+  end
+
+  def handle_special(data)
+    print(data) unless ignoring
+  end
+end
+
+p = GoLiveRemover.new(true, true)
+ARGF.each_line { |line| p.feed(line) }

data/demo/ebaySearch.rb

@@ -0,0 +1,93 @@
+require 'html/tree'
+require 'net/http'
+
+# A demo script showing HTML parsing after a HTTP request.
+# This does an eBay search for the given term(s), and displays the
+# results as a text table, delimited with '|' characters.
+#
+# usage:
+#   ruby ebaySearch.rb searchterm [...]
+#
+# If you give the -d flag, it contacts http://localhost:8080 instead
+# (for testing).
+#
+# Note that actually using this script is in violation of the
+# eBay User Agreement.
+#
+# A real robot would respect the REP.
+#
+# This is just an example.
+
+verbose = false
+if ARGV[0] == '-v'
+  verbose = true
+  ARGV.shift 
+end
+
+unless ARGV.size > 0
+  puts "usage: #{$0} [-v] searchterm [...]"
+  puts "  -v  turns on verbose error reporting"
+  exit 2
+end
+
+query = ARGV.join('+')
+
+queryHost = "search.ebay.com"
+queryPort = 80
+queryURL = "/search/search.dll" +
+  "?MfcISAPICommand=GetResult&ht=1&SortProperty=MetaEndSort&query=#{query}"
+
+# try to look like a real browser (don't know if it matters)...
+headers = {
+  'User-Agent' => 'Mozilla/5.0 (compatible; Konqueror/3.0.0-10; Linux)',
+  'Pragma' => 'no-cache',
+  'Cache-control' => 'no-cache',
+  'Accept' => 'text/*, image/jpeg, image/png, image/*, */*',
+  'Accept-Encoding' => 'x-gzip, gzip, identity',
+  'Accept-Charset' => 'ISO-8859-1',
+}
+
+data = ""
+
+# add these non-HTML 4.0 tags because eBay seems to use them
+#                (name,      is_block, is_inline, is_empty, can_omit)
+HTML::Tag.add_tag('ilayer', true, false, true, true)
+HTML::Tag.add_tag('layer', true, false, true, true)
+HTML::Tag.add_tag('nolayer', true, false, true, true)
+HTML::Tag.add_tag('noframe', true, false, false, false)
+
+begin
+  Net::HTTP.version_1_1
+  http = Net::HTTP.new(queryHost, queryPort)
+  # http.open_timeout = 30
+  http.read_timeout = 120
+  resp, data = http.get(queryURL, headers)
+rescue
+  print 'error:'
+  puts http.inspect
+  puts resp.inspect
+  exit 1
+end
+
+
+p = HTMLTree::Parser.new(verbose, false)
+p.feed(data)
+
+# Find all ViewItem links. These are in table rows for each item.
+itemAnchors = p.html.select { |ea| ea.tag == 'a' && ea['href'] =~ /ViewItem/ }
+
+# Now find their rows by going up to the first <tr>
+itemRows = itemAnchors.collect { |ea|
+  while ea.tag != 'tr'
+    ea = ea.parent
+  end
+  ea
+}
+
+# print the text from them
+itemRows.each { |row|
+  texts = row.select { |item| item.data?  }.  # just look at cdata
+    collect { |data| data.strip }.            # strip it
+    select { |data| data.size > 0 }           # and keep the non-blank fields
+  puts texts.join('|')
+}

data/demo/xpath.rb

@@ -0,0 +1,62 @@
+#!/usr/bin/ruby
+# This is a demo program that takes a given HTML file, parses it,
+# and allows exploration of XPath queries.
+require 'html/xmltree'
+require 'html/rexml-nodepath'
+require 'rexml/xpath'
+
+
+def displayXPath(d, path)
+  REXML::XPath.each(d, path) do |node|
+    puts node.full_path + " --> " + node.to_s
+  end
+  nil
+end
+
+
+unless ARGV.size >= 1
+  $stderr.puts(%Q{usage: #{$0} file.html                  interactive
+       #{$0} file.html expressions      read expressions from expressions
+})
+  exit(1)
+end
+
+$use_readline = false
+if $stderr.isatty
+  $stdout.sync = true
+  begin
+    require 'readline'
+    $use_readline = true
+    $stderr.puts('line editing enabled')
+    trap('SIGINT', 'IGNORE')
+  rescue LoadError
+    $use_readline = false
+  end
+end
+
+def getline(prompt)
+  if $use_readline
+    Readline.readline(prompt, true)
+  else
+    $stdout.print prompt
+    $stdin.gets
+  end
+end
+
+inputFile = ARGV.shift
+p = HTMLTree::XMLParser.new(true)
+p.parse_file_named(inputFile)
+d = p.document
+
+if ARGV.size > 0 then
+  for path in ARGV
+    displayXPath(d, path)
+  end
+  exit
+end
+
+prompt = 'Enter XPath expression on a single line (ctrl-D (unix) or ctrl-Z (win) to quit): '
+while expr = getline(prompt)
+    displayXPath(d, expr)
+    prompt = 'expr: '
+end