RubyGems - htmltools - Versions diffs - 1.10 - Mend

htmltools 1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

data/lib/html/tree.rb ADDED

@@ -0,0 +1,140 @@
+#!/usr/bin/ruby
+# This is an HTML parser that builds an element tree for further
+# processing. Attributes and data are also stored.
+#
+# Typical usage is:
+#   parser = HTMLTree::Parser.new(false, false)
+#   parser.parse_file_named('whatever.html')
+#   # then you have the tree built..
+#   parser.tree.dump
+#
+# Copyright::   Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>,
+#               Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+# License::   Ruby's
+# CVS ID::    $Id: tree.rb,v 1.2 2004/09/24 23:28:55 jhannes Exp $
+require 'html/tags'
+require 'html/stparser'
+require 'html/element'
+# This is a tree building HTML parser.
+module HTMLTree
+  class Parser < HTML::StackingParser
+    # verbose::  if true, will warn to $stderr on unknown
+    # tags/entities/characters, as well as missing end tags and extra end
+    # tags.
+    # strip_white:: if true, remove all non-essential whitespace. Note
+    # that there are browser bugs that may cause this to change the
+    # appearance of HTML (even though it shouldn't by the standard).
+    def initialize(verbose=false, strip_white=true)
+      super
+      reset
+    end
+    # Reset this parser so that it can parse a new document.
+    def reset
+      super
+      @rootNode = @currentNode = Document.new
+    end
+    # Return the tree that was built. This will be an HTMLTree::Element that
+    # represents the whole document. The \<html> node is a child of this.
+    def tree
+      @rootNode
+    end
+    # Return the <html> node, if any.
+    def html
+      @rootNode.html_node()
+    end
+    # no user-serviceable parts inside...
+    # though you can subclass carefully.
+    private
+    def add_child_to_current(tag, attrs)
+      node = Element.new(@currentNode, tag)
+      attrs.each { |a| node.add_attribute(*a) }
+      node
+    end
+    # callbacks
+    # add a child to the current node and descend
+    def handle_start_tag(tag, attrs)
+      node = add_child_to_current(tag, attrs)
+      @rootNode = node unless @rootNode
+      @currentNode = node
+    end
+    # go up to parent
+    def handle_end_tag(tag)
+      @currentNode = @currentNode.parent
+    end
+    # add a child to the current node
+    def handle_empty_tag(tag, attrs)
+      add_child_to_current(tag, attrs)
+    end
+    # Add a child to the current node and descend
+    # Assume that the unknown tag has an end tag.
+    def handle_unknown_tag(tag, attrs)
+      super
+      handle_start_tag(tag, attrs)
+    end
+    # go up to parent
+    def handle_missing_end_tag(tag)
+      super
+      handle_end_tag(tag)
+    end
+    # ignore
+    def handle_extra_end_tag(tag)
+      super
+    end
+    def handle_cdata(data)
+      node = Data.new(@currentNode, data)
+    end
+    def handle_script(data)
+      node = Data.new(@currentNode, data)
+    end
+    def handle_unknown_character(name)
+      node = Data.new(@currentNode, "&##{name};")
+    end
+    def handle_unknown_entity(name)
+      node = Data.new(@currentNode, "&#{name};")
+    end
+    def handle_comment(data)
+      super # make sure and strip whitespace.
+      node = Comment.new(@currentNode, data)
+    end
+    def handle_special(data)
+      node = HTMLTree::Special.new(@currentNode, data)
+      $stderr.print('special ', node, ' discarded') unless @currentNode
+    end
+  end
+end
+if $0 == __FILE__
+  $stdout.sync = true
+  class TestStackingParser < HTMLTree::Parser
+    $DEBUG = false
+    p = TestStackingParser.new(true, false)
+    p.parse_file_named(ARGV[0] || 'ebay.html')
+    File.open('xx.html', 'w') { |of|
+      p.tree.write(of)
+    }
+    p.tree.dump
+  end
+end

data/lib/html/xmltree.rb ADDED

@@ -0,0 +1,173 @@
+#!/usr/bin/ruby
+# This is a tree building HTML parser that makes an XML structure
+# using the format of REXML.
+#
+# Typical usage is:
+#   parser = HTMLTree::XMLParser.new(false, false)
+#   parser.parse_file_named('whatever.html')
+#   # then you have the tree built..
+#   parser.document # is a REXML::Document
+#
+# Copyright::   Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>,
+#               Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+# License::   Ruby's
+# CVS ID::    $Id: xmltree.rb,v 1.2 2004/09/24 23:28:55 jhannes Exp $
+require 'html/tags'
+require 'html/stparser'
+require 'rexml/element'
+require 'rexml/document'
+# REXML::Child
+#   REXML::XMLDecl
+#   REXML::Instruction
+#   REXML::Text
+#   REXML::Comment
+#   REXML::Entity
+#   REXML::Parent
+#     REXML::Element (+REXML::Namespace)
+#       REXML::Document
+#     REXML::DocType
+#
+# This is a tree building HTML parser that makes XML.
+module HTMLTree
+  class XMLParser < HTML::StackingParser
+    # verbose::  if true, will warn to $stderr on unknown
+    # tags/entities/characters, as well as missing end tags and extra end
+    # tags.
+    # strip_white:: if true, remove all non-essential whitespace. Note
+    # that there are browser bugs that may cause this to change the
+    # appearance of HTML (even though it shouldn't by the standard).
+    def initialize(verbose=false, strip_white=true)
+      super
+      reset
+    end
+    # Reset this parser so that it can parse a new document.
+    def reset
+      super
+      @rootNode = @currentNode = REXML::Document.new()
+    end
+    # Return the document that was built. This will be an
+    # REXML::Document that represents the whole document. The \<html>
+    # node is a child of this.
+    def document
+      @rootNode
+    end
+    def tree
+      document
+    end
+    # Return the root of the document, if any.
+    def root
+      @rootNode.root()
+    end
+    # Return the <html> node, if any.
+    def html
+      @rootNode.root.elements['html']
+    end
+    # no user-serviceable parts inside...
+    # though you can subclass carefully.
+    private
+    def add_child_to_current(tag, attrs)
+      node = REXML::Element.new(tag, @currentNode)
+      attrs.each { |a| node.attributes[a[0]] = a[1] }
+      node
+    end
+    # callbacks
+    # add a child to the current node and descend
+    def handle_start_tag(tag, attrs)
+      node = add_child_to_current(tag, attrs)
+      @rootNode = node unless @rootNode
+      @currentNode = node
+    end
+    # go up to parent
+    def handle_end_tag(tag)
+      @currentNode = @currentNode.parent
+    end
+    # add a child to the current node
+    def handle_empty_tag(tag, attrs)
+      add_child_to_current(tag, attrs)
+    end
+    # Add a child to the current node and descend
+    # Assume that the unknown tag has an end tag.
+    def handle_unknown_tag(tag, attrs)
+      super
+      handle_start_tag(tag, attrs)
+    end
+    # go up to parent
+    def handle_missing_end_tag(tag)
+      super
+      handle_end_tag(tag)
+    end
+    # ignore
+    def handle_extra_end_tag(tag)
+      super
+    end
+    def handle_cdata(data)
+      node = REXML::Text.new(data, !@stripWhitespace, @currentNode)
+      node.parent = @currentNode
+    end
+    def handle_script(data)
+      node = REXML::Comment.new(data, @currentNode)
+      node.parent = @currentNode
+    end
+    def handle_unknown_character(name)
+      node = REXML::Text.new("&##{name};", false, @currentNode)
+      node.raw = true
+      node.parent = @currentNode
+      node
+    end
+    def handle_unknown_entity(name)
+      node = REXML::Text.new("&#{name};", false, @currentNode)
+      node.raw = true
+      node.parent = @currentNode
+      node
+    end
+    def handle_comment(data)
+      super # strip white
+      node = REXML::Comment.new(data, @currentNode)
+      node.parent = @currentNode
+      node
+    end
+    def handle_special(data)
+      source = REXML::SourceFactory::create_from( "<#{data}>" )
+      node = REXML::DocType.new(source, @currentNode)
+      node.parent = @currentNode
+      node
+    end
+  end
+end
+if $0 == __FILE__
+  $stdout.sync = true
+  class TestStackingParser < HTMLTree::XMLParser
+    $DEBUG = false
+    p = TestStackingParser.new(true, false)
+    p.parse_file_named(ARGV[0] || 'ebay.html')
+    File.open('xx.html', 'w') { |of|
+      p.document.write(of)
+    }
+  end
+end

data/lib/html/xpath.rb ADDED

@@ -0,0 +1,72 @@
+# This module adapts REXML's XPath functionality for use with
+# <tt>HTMLTree::Parser</tt>.
+#
+# Copyright::   Copyright (C) 2003, Johannes Brodwall <johannes@brodwall.com>,
+#               Copyright (C) 2002, Ned Konz <ned@bike-nomad.com>
+# License::     Same as Ruby's
+# CVS ID:       $Id: xpath.rb,v 1.3 2004/09/24 23:28:55 jhannes Exp $
+require 'html/tree'
+require 'rexml/element'
+require 'rexml/document'
+require 'rexml/xpath'
+module HTMLTree
+  module TreeElement
+    # Given the XPath path, return an Array of matching sub-elements of
+    # the REXML tree.
+    def rexml_match(path)
+      node = as_rexml_document
+      REXML::XPath.match(node, path)
+    end
+  end
+  class Element
+    # convert the given HTMLTree::Element (or HTMLTree::Document) into
+    # a REXML::Element or REXML::Document, ready to use REXML::XPath on.
+    # Note that this caches the tree; further changes to my tree will
+    # not be reflected in subsequent calls
+    def as_rexml_document(rparent = nil, context = {})
+      return @_rexml_tree if @_rexml_tree
+      node = REXML::Element.new( tag, rparent, context )
+      attribute_order().each { |attr|
+        node.add_attribute(attr, attribute(attr).to_s)
+      }
+      children().each { |child|
+        childNode = child.as_rexml_document(node, context)
+      }
+      @_rexml_tree = node
+    end
+  end
+  class Data
+    def as_rexml_document(rparent = nil, context = {})
+      rparent.add_text(@_content)
+    end
+  end
+  class Comment
+    def as_rexml_document(rparent = nil, context = {})
+      node = REXML::Comment.new(@_content, parent)
+    end
+  end
+  class Special
+    def as_rexml_document(rparent = nil, context = {})
+      node = REXML::Instruction.new(@_content,
+        context[:respect_whitespace] || false, rparent)
+    end
+  end
+  class Document
+    def as_rexml_document(context = {})
+      node = REXML::Document.new(nil, context)
+      # add DocType
+      # add <HTML> node
+      html_node.as_rexml_document(node, context)
+      node
+    end
+  end
+end

data/test/suite.rb ADDED

@@ -0,0 +1,5 @@
+require 'test/tc_html-element.rb'
+require 'test/tc_html-tree.rb'
+require 'test/tc_stacking-parser.rb'
+require 'test/tc_xpath.rb'

data/test/tc_html-element.rb ADDED

@@ -0,0 +1,73 @@
+#!/usr/bin/ruby
+require 'html/element'
+require 'test/unit'
+module HTMLTree
+  class Element
+    def inspect
+      "<#{@_tag}> " + attributes.inspect + children.inspect
+    end
+  end
+end
+class HTMLElementTestCase < Test::Unit::TestCase
+  def setup
+    @e = HTMLTree::Element.new
+  end
+  attr_reader :e
+  def test_empty
+    assert_equal(nil, e.tag)
+    assert_equal({}, e.attributes)
+    assert_equal([], e.children)
+  end
+  def test_tag
+    e2 = HTMLTree::Element.new(nil, 'sometag')
+    assert_equal('sometag', e2.tag)
+    assert_equal({}, e2.attributes)
+    assert_equal([], e2.children)
+  end
+  def test_attribute
+    e.add_attribute('a', 'b')
+    assert_equal('b', e.attribute('a'))
+    assert_equal('b', e['a'])
+    e.add_attribute('a', 'c')
+    assert_equal(['b','c'], e.attribute('a'))
+    assert_equal(['b','c'], e['a'])
+    e.add_attribute('a', 'd', 'e')
+    assert_equal(['b','c', 'd', 'e'], e.attribute('a'))
+    e.add_attribute('b', ['c','d'])
+    assert_equal(['c','d'], e.attribute('b'))
+    e.add_attribute('b', ['e','f'])
+    assert_equal(['c','d', 'e', 'f'], e.attribute('b'))
+    e['b'] = 'aaa'
+    assert_equal('aaa', e.attribute('b'))
+  end
+  def test_parent
+    p = HTMLTree::Element.new(nil, 'p')
+    c = HTMLTree::Element.new(p, 'c')
+    assert_equal(nil, p.parent)
+    assert_equal([c], p.children)
+    assert_equal(p, c.parent)
+    d = HTMLTree::Element.new(p, 'd')
+    assert_equal([c,d], p.children)
+    assert_equal(p, d.parent)
+    p.remove_child(d)
+    assert_equal([c], p.children)
+    assert_equal(p, c.parent)
+    assert_equal(nil, d.parent)
+  end
+  def test_iterator
+    p = HTMLTree::Element.new(nil, 'p')
+    c = HTMLTree::Element.new(p, 'c')
+    d = HTMLTree::Element.new(p, 'd')
+  end
+end