RubyGems - htmltools - Versions diffs - 1.10 - Mend

htmltools 1.10

Files changed (20) hide show

@@ -0,0 +1,140 @@
+#!/usr/bin/ruby
+# This is an HTML parser that builds an element tree for further
+# processing. Attributes and data are also stored.
+#
+# Typical usage is:
+#   parser = HTMLTree::Parser.new(false, false)
+#   parser.parse_file_named('whatever.html')
+#   # then you have the tree built..
+#   parser.tree.dump
+#
+# Copyright::   Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>,
+#               Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+# License::   Ruby's
+# CVS ID::    $Id: tree.rb,v 1.2 2004/09/24 23:28:55 jhannes Exp $
+require 'html/tags'
+require 'html/stparser'
+require 'html/element'
+# This is a tree building HTML parser.
+module HTMLTree
+  class Parser < HTML::StackingParser
+    # verbose::  if true, will warn to $stderr on unknown
+    # tags/entities/characters, as well as missing end tags and extra end
+    # tags.
+    # strip_white:: if true, remove all non-essential whitespace. Note
+    # that there are browser bugs that may cause this to change the
+    # appearance of HTML (even though it shouldn't by the standard).
+    def initialize(verbose=false, strip_white=true)
+      super
+      reset
+    end
+    # Reset this parser so that it can parse a new document.
+    def reset
+      super
+      @rootNode = @currentNode = Document.new
+    end
+    # Return the tree that was built. This will be an HTMLTree::Element that
+    # represents the whole document. The \<html> node is a child of this.
+    def tree
+      @rootNode
+    end
+    # Return the <html> node, if any.
+    def html
+      @rootNode.html_node()
+    end
+    # no user-serviceable parts inside...
+    # though you can subclass carefully.
+    private
+    def add_child_to_current(tag, attrs)
+      node = Element.new(@currentNode, tag)
+      attrs.each { |a| node.add_attribute(*a) }
+      node
+    end
+    # callbacks
+    # add a child to the current node and descend
+    def handle_start_tag(tag, attrs)
+      node = add_child_to_current(tag, attrs)
+      @rootNode = node unless @rootNode
+      @currentNode = node
+    end
+    # go up to parent
+    def handle_end_tag(tag)
+      @currentNode = @currentNode.parent
+    end
+    # add a child to the current node
+    def handle_empty_tag(tag, attrs)
+      add_child_to_current(tag, attrs)
+    end
+    # Add a child to the current node and descend
+    # Assume that the unknown tag has an end tag.
+    def handle_unknown_tag(tag, attrs)
+      super
+      handle_start_tag(tag, attrs)
+    end
+    # go up to parent
+    def handle_missing_end_tag(tag)
+      super
+      handle_end_tag(tag)
+    end
+    # ignore
+    def handle_extra_end_tag(tag)
+      super
+    end
+    def handle_cdata(data)
+      node = Data.new(@currentNode, data)
+    end
+    def handle_script(data)
+      node = Data.new(@currentNode, data)
+    end
+    def handle_unknown_character(name)
+      node = Data.new(@currentNode, "&##{name};")
+    end
+    def handle_unknown_entity(name)
+      node = Data.new(@currentNode, "&#{name};")
+    end
+    def handle_comment(data)
+      super # make sure and strip whitespace.
+      node = Comment.new(@currentNode, data)
+    end
+    def handle_special(data)
+      node = HTMLTree::Special.new(@currentNode, data)
+      $stderr.print('special ', node, ' discarded') unless @currentNode
+    end
+  end
+end
+if $0 == __FILE__
+  $stdout.sync = true
+  class TestStackingParser < HTMLTree::Parser
+    $DEBUG = false
+    p = TestStackingParser.new(true, false)
+    p.parse_file_named(ARGV[0] || 'ebay.html')
+    File.open('xx.html', 'w') { |of|
+      p.tree.write(of)
+    }
+    p.tree.dump
+  end
+end

data/lib/html/xmltree.rb ADDED

@@ -0,0 +1,173 @@
+#!/usr/bin/ruby
+# This is a tree building HTML parser that makes an XML structure
+# using the format of REXML.
+#
+# Typical usage is:
+#   parser = HTMLTree::XMLParser.new(false, false)
+#   parser.parse_file_named('whatever.html')
+#   # then you have the tree built..
+#   parser.document # is a REXML::Document
+#
+# Copyright::   Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>,
+#               Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+# License::   Ruby's
+# CVS ID::    $Id: xmltree.rb,v 1.2 2004/09/24 23:28:55 jhannes Exp $
+require 'html/tags'
+require 'html/stparser'
+require 'rexml/element'
+require 'rexml/document'
+# REXML::Child
+#   REXML::XMLDecl
+#   REXML::Instruction
+#   REXML::Text
+#   REXML::Comment
+#   REXML::Entity
+#   REXML::Parent
+#     REXML::Element (+REXML::Namespace)
+#       REXML::Document
+#     REXML::DocType
+#
+# This is a tree building HTML parser that makes XML.
+module HTMLTree
+  class XMLParser < HTML::StackingParser
+    # verbose::  if true, will warn to $stderr on unknown
+    # tags/entities/characters, as well as missing end tags and extra end
+    # tags.
+    # strip_white:: if true, remove all non-essential whitespace. Note
+    # that there are browser bugs that may cause this to change the
+    # appearance of HTML (even though it shouldn't by the standard).
+    def initialize(verbose=false, strip_white=true)
+      super
+      reset
+    end
+    # Reset this parser so that it can parse a new document.
+    def reset
+      super
+      @rootNode = @currentNode = REXML::Document.new()
+    end
+    # Return the document that was built. This will be an
+    # REXML::Document that represents the whole document. The \<html>
+    # node is a child of this.
+    def document
+      @rootNode
+    end
+    def tree
+      document
+    end
+    # Return the root of the document, if any.
+    def root
+      @rootNode.root()
+    end
+    # Return the <html> node, if any.
+    def html
+      @rootNode.root.elements['html']
+    end
+    # no user-serviceable parts inside...
+    # though you can subclass carefully.
+    private
+    def add_child_to_current(tag, attrs)
+      node = REXML::Element.new(tag, @currentNode)
+      attrs.each { |a| node.attributes[a[0]] = a[1] }
+      node
+    end
+    # callbacks
+    # add a child to the current node and descend
+    def handle_start_tag(tag, attrs)
+      node = add_child_to_current(tag, attrs)
+      @rootNode = node unless @rootNode
+      @currentNode = node
+    end
+    # go up to parent
+    def handle_end_tag(tag)
+      @currentNode = @currentNode.parent
+    end
+    # add a child to the current node
+    def handle_empty_tag(tag, attrs)
+      add_child_to_current(tag, attrs)
+    end
+    # Add a child to the current node and descend
+    # Assume that the unknown tag has an end tag.
+    def handle_unknown_tag(tag, attrs)
+      super
+      handle_start_tag(tag, attrs)
+    end
+    # go up to parent
+    def handle_missing_end_tag(tag)
+      super
+      handle_end_tag(tag)
+    end
+    # ignore
+    def handle_extra_end_tag(tag)
+      super
+    end
+    def handle_cdata(data)
+      node = REXML::Text.new(data, !@stripWhitespace, @currentNode)
+      node.parent = @currentNode
+    end
+    def handle_script(data)
+      node = REXML::Comment.new(data, @currentNode)
+      node.parent = @currentNode
+    end
+    def handle_unknown_character(name)
+      node = REXML::Text.new("&##{name};", false, @currentNode)
+      node.raw = true
+      node.parent = @currentNode
+      node
+    end
+    def handle_unknown_entity(name)
+      node = REXML::Text.new("&#{name};", false, @currentNode)
+      node.raw = true
+      node.parent = @currentNode
+      node
+    end
+    def handle_comment(data)
+      super # strip white
+      node = REXML::Comment.new(data, @currentNode)
+      node.parent = @currentNode
+      node
+    end
+    def handle_special(data)
+      source = REXML::SourceFactory::create_from( "<#{data}>" )
+      node = REXML::DocType.new(source, @currentNode)
+      node.parent = @currentNode
+      node
+    end
+  end
+end
+if $0 == __FILE__
+  $stdout.sync = true
+  class TestStackingParser < HTMLTree::XMLParser
+    $DEBUG = false
+    p = TestStackingParser.new(true, false)
+    p.parse_file_named(ARGV[0] || 'ebay.html')
+    File.open('xx.html', 'w') { |of|
+      p.document.write(of)
+    }
+  end
+end

data/lib/html/xpath.rb ADDED

@@ -0,0 +1,72 @@
+# This module adapts REXML's XPath functionality for use with
+# <tt>HTMLTree::Parser</tt>.
+#
+# Copyright::   Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>,
+#               Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+# License::     Same as Ruby's
+# CVS ID:       $Id: xpath.rb,v 1.3 2004/09/24 23:28:55 jhannes Exp $
+require 'html/tree'
+require 'rexml/element'
+require 'rexml/document'
+require 'rexml/xpath'
+module HTMLTree
+  module TreeElement
+    # Given the XPath path, return an Array of matching sub-elements of
+    # the REXML tree.
+    def rexml_match(path)
+      node = as_rexml_document
+      REXML::XPath.match(node, path)
+    end
+  end
+  class Element
+    # convert the given HTMLTree::Element (or HTMLTree::Document) into
+    # a REXML::Element or REXML::Document, ready to use REXML::XPath on.
+    # Note that this caches the tree; further changes to my tree will
+    # not be reflected in subsequent calls
+    def as_rexml_document(rparent = nil, context = {})
+      return @_rexml_tree if @_rexml_tree
+      node = REXML::Element.new( tag, rparent, context )
+      attribute_order().each { |attr|
+        node.add_attribute(attr, attribute(attr).to_s)
+      }
+      children().each { |child|
+        childNode = child.as_rexml_document(node, context)
+      }
+      @_rexml_tree = node
+    end
+  end
+  class Data
+    def as_rexml_document(rparent = nil, context = {})
+      rparent.add_text(@_content)
+    end
+  end
+  class Comment
+    def as_rexml_document(rparent = nil, context = {})
+      node = REXML::Comment.new(@_content, parent)
+    end
+  end
+  class Special
+    def as_rexml_document(rparent = nil, context = {})
+      node = REXML::Instruction.new(@_content,
+        context[:respect_whitespace] || false, rparent)
+    end
+  end
+  class Document
+    def as_rexml_document(context = {})
+      node = REXML::Document.new(nil, context)
+      # add DocType
+      # add <HTML> node
+      html_node.as_rexml_document(node, context)
+      node
+    end
+  end
+end

data/test/suite.rb ADDED

@@ -0,0 +1,5 @@
+require 'test/tc_html-element.rb'
+require 'test/tc_html-tree.rb'
+require 'test/tc_stacking-parser.rb'
+require 'test/tc_xpath.rb'

data/test/tc_html-element.rb ADDED

@@ -0,0 +1,73 @@
+#!/usr/bin/ruby
+require 'html/element'
+require 'test/unit'
+module HTMLTree
+  class Element
+    def inspect
+      "<#{@_tag}> " + attributes.inspect + children.inspect
+    end
+  end
+end
+class HTMLElementTestCase < Test::Unit::TestCase
+  def setup
+    @e = HTMLTree::Element.new
+  end
+  attr_reader :e
+  def test_empty
+    assert_equal(nil, e.tag)
+    assert_equal({}, e.attributes)
+    assert_equal([], e.children)
+  end
+  def test_tag
+    e2 = HTMLTree::Element.new(nil, 'sometag')
+    assert_equal('sometag', e2.tag)
+    assert_equal({}, e2.attributes)
+    assert_equal([], e2.children)
+  end
+  def test_attribute
+    e.add_attribute('a', 'b')
+    assert_equal('b', e.attribute('a'))
+    assert_equal('b', e['a'])
+    e.add_attribute('a', 'c')
+    assert_equal(['b','c'], e.attribute('a'))
+    assert_equal(['b','c'], e['a'])
+    e.add_attribute('a', 'd', 'e')
+    assert_equal(['b','c', 'd', 'e'], e.attribute('a'))
+    e.add_attribute('b', ['c','d'])
+    assert_equal(['c','d'], e.attribute('b'))
+    e.add_attribute('b', ['e','f'])
+    assert_equal(['c','d', 'e', 'f'], e.attribute('b'))
+    e['b'] = 'aaa'
+    assert_equal('aaa', e.attribute('b'))
+  end
+  def test_parent
+    p = HTMLTree::Element.new(nil, 'p')
+    c = HTMLTree::Element.new(p, 'c')
+    assert_equal(nil, p.parent)
+    assert_equal([c], p.children)
+    assert_equal(p, c.parent)
+    d = HTMLTree::Element.new(p, 'd')
+    assert_equal([c,d], p.children)
+    assert_equal(p, d.parent)
+    p.remove_child(d)
+    assert_equal([c], p.children)
+    assert_equal(p, c.parent)
+    assert_equal(nil, d.parent)
+  end
+  def test_iterator
+    p = HTMLTree::Element.new(nil, 'p')
+    c = HTMLTree::Element.new(p, 'c')
+    d = HTMLTree::Element.new(p, 'd')
+  end
+end