xmlscan 0.2.3

data/THANKS

@@ -0,0 +1,11 @@
+$Id: THANKS,v 1.4.2.1 2003/02/15 13:22:01 katsu Exp $
+
+Thanks to all of the following for their valuable hints, fixes,
+discussions, and contributions:
+
+Yoshida Masato <yoshidam@yoshidam.net>
+TAKAHASHI Masayoshi <maki@inac.co.jp>
+NAKAMURA, Hiroshi <nakahiro@sarion.co.jp>
+James Britt <james@jamesbritt.com>
+Takaaki Tateishi <ttate@kt.jaist.ac.jp>
+Tanaka Akira <akr@m17n.org>

data/VERSION

@@ -0,0 +1 @@
+0.2.3

data/install.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/ruby
+#
+# install.rb
+#
+# $Id: install.rb,v 1.2 2002/12/26 21:09:38 katsu Exp $
+
+require 'rbconfig'
+require 'ftools'
+require 'find'
+require 'getoptlong'
+
+DEFAULT_DESTDIR = Config::CONFIG['sitelibdir'] || Config::CONFIG['sitedir']
+SRCDIR = File.dirname(__FILE__)
+
+
+def install_rb(from, to)
+  from = SRCDIR + '/' + from
+  Find.find(from) { |src|
+    next unless File.file? src
+    next unless /\.rb\z/ =~ src
+    dst = src.sub(/\A#{Regexp.escape(from)}/, to)
+    File.makedirs File.dirname(dst), true
+    File.install src, dst, 0644, true
+  }
+end
+
+
+destdir = DEFAULT_DESTDIR
+begin
+  GetoptLong.new([ "-d", "--destdir", GetoptLong::REQUIRED_ARGUMENT ]
+                ).each_option { |opt, arg|
+    case opt
+    when '-d' then
+      destdir = arg
+    end
+  }
+rescue
+  exit 2
+end
+
+install_rb "lib", destdir

data/lib/xmlscan/htmlscan.rb

@@ -0,0 +1,290 @@
+# encoding: UTF-8
+#
+# xmlscan/htmlscan.rb
+#
+#   Copyright (C) Ueno Katsuhiro 2002
+#
+# $Id: htmlscan.rb,v 1.16.2.2 2003/05/01 15:43:23 katsu Exp $
+#
+
+require 'xmlscan/scanner'
+
+
+module XMLScan
+
+  class HTMLScanner < XMLScanner
+
+    private
+
+    def wellformed_error(msg)
+      # All wellformed error raised by XMLScanner are ignored.
+      # XMLScanner only raises wellformed error in stan_stag, which is a
+      # method completely overrided by HTMLScanner, so this method is
+      # never called in fact.
+    end
+
+    def on_xmldecl
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_version(str)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_encoding(str)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_standalone(str)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_other(name, value)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_end
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_stag_end_empty(name)
+      raise "[BUG] this method must be never called"
+    end
+
+
+    private
+
+    def scan_comment(s)
+      s[0,4] = ''  # remove `<!--'
+      comm = ''
+      until /--/n =~ s
+        comm << s
+        s = @src.get_plain
+        unless s then
+          parse_error "unterminated comment meets EOF"
+          return on_comment(comm)
+        end
+      end
+      comm << $`
+      s = $'
+      until s.empty? || s.strip.empty? and @src.close_tag   # --> or -- >
+        comm << '--'
+        if /\A\s*--/n =~ s then   # <!--hoge-- --
+          comm << $&
+          s = $'
+          if s.empty? and @src.close_tag then   # <!--hoge-- -->
+            parse_error "`-->' is found but comment must not end here"
+            comm.chop!.chop!
+            break
+          end
+        else                     # <!--hoge-- fuga
+          parse_error "only whitespace can appear between two comments"
+        end
+        if /\A-\s*\z/n =~ s and @src.close_tag then  # <!--hoge--->
+          parse_error "`-->' is found but comment must not end here"
+          comm.chop!
+          break
+        end
+        until /--/n =~ s      # copy & paste for performance
+          comm << s
+          s = @src.get_plain
+          unless s then
+            parse_error "unterminated comment meets EOF"
+            return on_comment(comm)
+          end
+        end
+        comm << $`
+        s = $'
+      end
+      on_comment comm
+    end
+
+
+    alias scan_xml_pi  scan_pi    # PIO "<?" PIC "?>"  -- <? PI ?> --
+
+
+    def scan_pi(s)   # <?PI >  this is default in SGML.
+      s[0,2] = ''    # remove `<?'
+      pi = s
+      until @src.close_tag
+        s = @src.get_plain
+        unless s then
+          parse_error "unterminated PI meets EOF"
+          break
+        end
+        pi << s
+      end
+      on_pi '', pi
+    end
+
+
+    def scan_stag(s)
+      unless /(?=[\/\s='"])/n =~ s then
+        name = s
+        name[0,1] = ''        # remove `<'
+        if name.empty? then   # <> or <<
+          if @src.close_tag then
+            return found_empty_stag
+          else
+            parse_error "parse error at `<'"
+            return on_chardata('<')
+          end
+        end
+        on_stag name
+        found_unclosed_stag name unless @src.close_tag
+        on_stag_end name
+      else
+        name = $`
+        s = $'
+        name[0,1] = ''        # remove `<'
+        if name.empty? then   # `< tag' or `<=`
+          parse_error "parse error at `<'"
+          if @src.close_tag then
+            s << '>'
+          end
+          return on_chardata('<' << s)
+        end
+        on_stag name
+        begin
+          continue = false
+          s.scan(
+         /([^\s=\/'"]+)(?:\s*=\s*(?:('[^']*'?|"[^"]*"?)|([^\s='"]+)))?|(\S)/n
+                 ) { |key,val,val2,error|
+            if key then
+              if val then                # key="value"
+                on_attribute key
+                qmark = val.slice!(0,1)
+                if val[-1] == qmark[0] then
+                  val.chop!
+                  scan_attvalue val unless val.empty?
+                else
+                  scan_attvalue val unless val.empty?
+                  begin
+                    s = @src.get
+                    unless s then
+                      parse_error "unterminated attribute `#{key}' meets EOF"
+                      break
+                    end
+                    c = s[0]
+                    val, s = s.split(qmark, 2)
+                    scan_attvalue '>' unless c == ?< or c == ?>
+                    scan_attvalue val if c
+                  end until s
+                  continue = s
+                end
+                on_attribute_end key
+              elsif val2 then            # key=value
+                on_attribute key
+                on_attr_value val2
+                on_attribute_end key
+              else                       # value
+                on_attribute nil
+                on_attr_value key
+                on_attribute_end nil
+              end
+            else
+              parse_error "parse error at `#{error}'"
+            end
+          }
+        end while continue
+        found_unclosed_stag name unless @src.close_tag
+        on_stag_end name
+      end
+    end
+
+
+    # This method should be called only from on_stag_end.
+    def get_cdata_content
+      unless not s = @src.test or s[0] == ?< && s[1] == ?/ then
+        dst = @src.get
+        until not s = @src.test or s[0] == ?< && s[1] == ?/
+          dst << @src.get_plain
+        end
+        dst
+      else
+        ''
+      end
+    end
+    public :get_cdata_content
+
+
+    def scan_bang_tag(s)
+      if s == '<!' and @src.close_tag then    # <!>
+        on_comment ''
+      else
+        parse_error "parse error at `<!'"
+        while s and not @src.close_tag        # skip entire
+          s = @src.get_plain
+        end
+      end
+    end
+
+
+    def scan_internal_dtd(s)
+      parse_error "DTD subset is found but it is not permitted in HTML"
+      skip_internal_dtd s
+    end
+
+
+    def found_invalid_pubsys(pubsys)
+      s = pubsys.upcase
+      return s if s == 'PUBLIC' or s == 'SYSTEM'
+      super
+    end
+
+
+    def scan_prolog(s)
+      doctype = 0
+      while s
+        if s[0] == ?< then
+          if (c = s[1]) == ?! then
+            if s[2] == ?- and s[3] == ?- then
+              scan_comment s
+            elsif /\A<!doctype(?=\s)/in =~ s then
+              doctype += 1
+              if doctype > 1 then
+                parse_error "another document type declaration is found"
+              end
+              scan_doctype $'
+            else
+              break
+            end
+          elsif c == ?? then
+            scan_pi s
+          else
+            break
+          end
+        elsif s.strip.empty? then
+          on_prolog_space s
+        else
+          break
+        end
+        s = @src.get
+      end
+      scan_content(s || @src.get)
+    end
+
+  end
+
+end
+
+
+
+
+
+if $0 == __FILE__ then
+  class TestVisitor
+    include XMLScan::Visitor
+    def parse_error(msg)
+      STDERR.printf("%s:%d: %s\n", $s.path, $s.lineno, msg) if $VERBOSE
+    end
+  end
+
+  $s = scan = XMLScan::HTMLScanner.new(TestVisitor.new)
+  src = ARGF
+  def src.path; filename; end
+  t1 = Time.times.utime
+  scan.parse src
+  t2 = Time.times.utime
+  STDERR.printf "%2.3f sec\n", t2 - t1
+end

data/lib/xmlscan/namespace.rb

@@ -0,0 +1,353 @@
+# encoding: UTF-8
+#
+# xmlscan/namespace.rb
+#
+#   Copyright (C) Ueno Katsuhiro 2002
+#
+# $Id: namespace.rb,v 1.13 2003/01/22 13:06:18 katsu Exp $
+#
+
+require 'xmlscan/parser'
+
+
+module XMLScan
+
+  class NSParseError < ParseError ; end
+  class NSNotWellFormedError < NotWellFormedError ; end
+  class NSNotValidError < NotValidError ; end
+
+
+  module NSVisitor
+
+    include Visitor
+
+    def ns_parse_error(msg)
+      raise NSParseError.new(msg)
+    end
+
+    def ns_wellformed_error(msg)
+      raise NSNotWellFormedError.new(msg)
+    end
+
+    def ns_valid_error(msg)
+      raise NSNotValidError.new(msg)
+    end
+
+    #
+    # <foo:bar hoge:fuga='' hoge=''  >
+    # <foo     hoge:fuga='' hoge=''  >
+    #  ^       ^          ^ ^     ^  ^
+    #  1       2          3 4     5  6
+    #
+    #  The following method will be called with the following arguments
+    #  when the parser reaches the above point;
+    #
+    #    1: on_stag_ns           ('foo:bar', 'foo', 'bar')
+    #        or
+    #       on_stag_ns           ('foo', '', 'foo')
+    #    2: on_attribute_ns      ('hoge:fuga', 'hoge', 'fuga')
+    #    3: on_attribute_end     ('hoge:fuga')
+    #    4: on_attribute_ns      ('hoge', nil, 'hoge')
+    #    5: on_attribute_end     ('hoge')
+    #    6: on_stag_end_ns       ('foo:bar', { 'foo' => '', ... })
+    #        or
+    #       on_stag_end_empty_ns ('foo:bar', { 'foo' => '', ... })
+    #
+
+    def on_stag_ns(qname, prefix, localpart)
+    end
+
+    def on_attribute_ns(qname, prefix, localpart)
+    end
+
+    def on_stag_end_ns(qname, namespaces)
+    end
+
+    def on_stag_end_empty_ns(qname, namespaces)
+    end
+
+  end
+
+
+
+
+  class XMLNamespaceDecoration < Decoration
+
+    proc {
+      h = {'foo'=>true} ; h['foo'] = nil
+      raise "requires Ruby-1.6 or above" unless h.key? 'foo'
+    }.call
+
+    PredefinedNamespace = {
+      'xml'   => 'http://www.w3.org/XML/1998/namespace',
+      'xmlns' => 'http://www.w3.org/2000/xmlns/',
+    }
+
+    ReservedNamespace = PredefinedNamespace.invert
+
+
+    def ns_parse_error(msg)
+      @orig_visitor.ns_parse_error msg
+    end
+
+    def ns_wellformed_error(msg)
+      @orig_visitor.ns_wellformed_error msg
+    end
+
+    def ns_valid_error(msg)
+      @orig_visitor.ns_valid_error msg
+    end
+
+
+    def on_start_document
+      @namespace = {} #PredefinedNamespace.dup
+      @ns_hist = []
+      @ns_undeclared = {}     # for checking undeclared namespace prefixes.
+      @prev_prefix = {}       # for checking doubled attributes.
+      @dont_same = []         # ditto.
+      @xmlns = NamespaceDeclaration.new(self)
+      @orig_visitor = @visitor
+      @visitor.on_start_document
+    end
+
+
+    def on_stag(name)
+      @ns_hist.push nil
+      unless /:/n =~ name then
+        @visitor.on_stag_ns name, '', name
+      else
+        prefix, localpart = $`, $'
+        if localpart.include? ?: then
+          ns_parse_error "localpart `#{localpart}' includes `:'"
+        end
+        if prefix == 'xmlns' then
+          ns_wellformed_error \
+            "prefix `xmlns' is not used for namespace prefix declaration"
+        end
+        unless @namespace.key? prefix then
+          if uri = PredefinedNamespace[prefix] then
+            @namespace[prefix] = uri
+          else
+            @ns_undeclared[prefix] = true
+          end
+        end
+        @visitor.on_stag_ns name, prefix, localpart
+      end
+    end
+
+
+    def on_attribute(name)
+      if /:/n =~ name then
+        prefix, localpart = $`, $'
+        if localpart.include? ?: then
+          ns_parse_error "localpart `#{localpart}' includes `:'"
+        end
+        unless @namespace.key? prefix then
+          if uri = PredefinedNamespace[prefix] then
+            @namespace[prefix] = uri
+          else
+            @ns_undeclared[prefix] = true
+          end
+        end
+        if prefix == 'xmlns' then
+          @visitor = @xmlns
+          @xmlns.on_xmlns_start localpart
+        else
+          if prev = @prev_prefix[localpart] then
+            @dont_same.push [ prev, prefix, localpart ]
+          end
+          @prev_prefix[localpart] = prefix
+          @visitor.on_attribute_ns name, prefix, localpart
+        end
+      elsif name == 'xmlns' then
+        @visitor = @xmlns
+        @xmlns.on_xmlns_start ''
+      else
+        @visitor.on_attribute_ns name, nil, name
+      end
+    end
+
+
+    class NamespaceDeclaration
+
+      include XMLScan::Visitor
+
+      def initialize(parent)
+        @parent = parent
+      end
+
+      def on_xmlns_start(prefix)
+        @prefix = prefix
+        @nsdecl = ''
+      end
+
+      def on_attr_value(str)
+        @nsdecl << str
+      end
+
+      def on_attr_entityref(ref)
+        @parent.ns_wellformed_error \
+          "xmlns includes undeclared entity reference"
+      end
+
+      def on_attr_charref(code)
+        @nsdecl << [code].pack('U')
+      end
+
+      def on_attr_charref_hex(code)
+        @nsdecl << [code].pack('U')
+      end
+
+      def on_attribute_end(name)
+        @parent.on_xmlns_end @prefix, @nsdecl
+      end
+
+    end
+
+
+    def on_xmlns_end(prefix, uri)
+      @visitor = @orig_visitor
+      if PredefinedNamespace.key? prefix then
+        if prefix == 'xmlns' then
+          ns_wellformed_error \
+            "prefix `xmlns' can't be bound to any namespace explicitly"
+        elsif (s = PredefinedNamespace[prefix]) != uri then
+          ns_wellformed_error \
+            "prefix `#{prefix}' can't be bound to any namespace except `#{s}'"
+        end
+      end
+      if uri.empty? then
+        if prefix.empty? then
+          uri = nil
+        else
+          ns_parse_error "`#{prefix}' is bound to empty namespace name"
+        end
+      elsif ReservedNamespace.key? uri then
+        unless (s = ReservedNamespace[uri]) == prefix then
+          ns_wellformed_error \
+            "namespace `#{uri}' is reserved for prefix `#{s}'"
+        end
+      end
+      (@ns_hist.last || @ns_hist[-1] = {})[prefix] = @namespace[prefix]
+      @namespace[prefix] = uri
+      @ns_undeclared.delete prefix
+    end
+
+
+    def fix_namespace
+      unless @ns_undeclared.empty? then
+        @ns_undeclared.each_key { |i|
+          @visitor.ns_wellformed_error "prefix `#{i}' is not declared"
+        }
+        @ns_undeclared.clear
+      end
+      unless @dont_same.empty? then
+        @dont_same.each { |n1,n2,l|
+          if @namespace[n1] == @namespace[n2] then
+            ns_wellformed_error \
+              "doubled localpart `#{l}' in the same namespace"
+          end
+        }
+        @dont_same.clear
+      end
+      @prev_prefix.clear
+    end
+
+
+    def on_stag_end(name)
+      fix_namespace
+      @visitor.on_stag_end_ns name, @namespace
+    end
+
+
+    def on_etag(name)
+      h = @ns_hist.pop and @namespace.update h
+      @visitor.on_etag name
+    end
+
+
+    def on_stag_end_empty(name)
+      fix_namespace
+      @visitor.on_stag_end_empty_ns name, @namespace
+      h = @ns_hist.pop and @namespace.update h
+    end
+
+
+    def on_doctype(root, pubid, sysid)
+      if root.count(':') > 1 then
+        ns_parse_error "qualified name `#{root}' includes `:'"
+      end
+      @visitor.on_doctype root, pubid, sysid
+    end
+
+
+    def on_pi(target, pi)
+      if target.include? ?: then
+        ns_parse_error "PI target `#{target}' includes `:'"
+      end
+      @visitor.on_pi target, pi
+    end
+
+
+    def on_entityref(ref)
+      if ref.include? ?: then
+        ns_parse_error "entity reference `#{ref}' includes `:'"
+      end
+      @visitor.on_entityref ref
+    end
+
+
+    def on_attr_entityref(ref)
+      if ref.include? ?: then
+        ns_parse_error "entity reference `#{ref}' includes `:'"
+      end
+      @visitor.on_attr_entityref ref
+    end
+
+  end
+
+
+
+  class XMLParserNS < XMLParser
+
+    def initialize(*)
+      super
+      @visitor = @decoration = XMLNamespaceDecoration.new(@visitor)
+    end
+
+  end
+
+end
+
+
+
+
+
+if $0 == __FILE__ then
+  class TestVisitor
+    include XMLScan::NSVisitor
+    def parse_error(msg)
+      STDERR.printf("%s:%d: %s\n", $s.path, $s.lineno, msg) if $VERBOSE
+    end
+    def wellformed_error(msg)
+      STDERR.printf("%s:%d: WFC: %s\n", $s.path, $s.lineno, msg) if $VERBOSE
+    end
+    def warning(msg)
+      STDERR.printf("%s:%d: warning: %s\n", $s.path,$s.lineno, msg) if $VERBOSE
+    end
+    def ns_parse_error(msg)
+      STDERR.printf("%s:%d: %s\n", $s.path, $s.lineno, msg) if $VERBOSE
+    end
+    def ns_wellformed_error(msg)
+      STDERR.printf("%s:%d: NSC: %s\n", $s.path, $s.lineno, msg) if $VERBOSE
+    end
+  end
+
+  $s = scan = XMLScan::XMLParserNS.new(TestVisitor.new)
+  src = ARGF
+  def src.path; filename; end
+  t1 = Time.times.utime
+  scan.parse src
+  t2 = Time.times.utime
+  STDERR.printf "%2.3f sec\n", t2 - t1
+end