rgen 0.3.0 → 0.4.0

data/redist/xmlscan/install.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/ruby
+#
+# install.rb
+#
+# $Id: install.rb,v 1.2 2002/12/26 21:09:38 katsu Exp $
+
+require 'rbconfig'
+require 'ftools'
+require 'find'
+require 'getoptlong'
+
+DEFAULT_DESTDIR = Config::CONFIG['sitelibdir'] || Config::CONFIG['sitedir']
+SRCDIR = File.dirname(__FILE__)
+
+
+def install_rb(from, to)
+  from = SRCDIR + '/' + from
+  Find.find(from) { |src|
+    next unless File.file? src
+    next unless /\.rb\z/ =~ src
+    dst = src.sub(/\A#{Regexp.escape(from)}/, to)
+    File.makedirs File.dirname(dst), true
+    File.install src, dst, 0644, true
+  }
+end
+
+
+destdir = DEFAULT_DESTDIR
+begin
+  GetoptLong.new([ "-d", "--destdir", GetoptLong::REQUIRED_ARGUMENT ]
+                ).each_option { |opt, arg|
+    case opt
+    when '-d' then
+      destdir = arg
+    end
+  }
+rescue
+  exit 2
+end
+
+install_rb "lib", destdir

data/redist/xmlscan/lib/xmlscan/encoding.rb

@@ -0,0 +1,311 @@
+#
+# xmlscan/encoding.rb
+#
+#   Copyright (C) Ueno Katsuhiro 2002
+#
+# $Id: encoding.rb,v 1.3 2003/01/12 04:10:33 katsu Exp $
+#
+
+require 'xmlscan/visitor'
+
+
+module XMLScan
+
+  class EncodingError < Error ; end
+
+
+  class Converter
+
+    def initialize
+    end
+
+    def convert(s)
+      s
+    end
+
+    def finish
+      ''
+    end
+
+  end
+
+
+
+  class SimpleConverter < Converter
+
+    def SimpleConverter.new_class(block)
+      Class.new(self).module_eval {
+        define_method(:convert, block)
+        self
+      }
+    end
+
+    # checking for Module#define_method works
+    begin
+      Class.new.module_eval{define_method(:a){};self}.new.a
+    rescue Exception
+      class << SimpleConverter
+        remove_method :new_class
+      end
+      def SimpleConverter.new_class(block)
+        Class.new(self).module_eval {
+          const_set :ConvProc, block
+          module_eval "def convert(s) ; ConvProc.call s ; end"
+          self
+        }
+      end
+    end
+
+  end
+
+
+
+  class EncodingClass
+
+    KCODE_None = //n.kcode
+
+
+    class ConverterProperty
+
+      def inspect
+        "#<Conversion #{@from.name}:#{@to.name} #{@cost}>"
+      end
+
+      def initialize(from, to, cost, klass = nil)
+        @from, @to, @cost, @klass = from, to, cost, klass
+      end
+
+      def new_converter
+        @klass and @klass.new
+      end
+
+      attr_reader :from, :to, :cost
+
+    end
+
+
+    class EncodingProperty
+
+      def inspect
+        s = "#<Encoding #{@name}/#{@kcode}>"
+      end
+
+      def initialize(name)
+        @name = name
+        conv = ConverterProperty.new(self, self, 0)
+        @converter = { self => conv }
+        @convertable_from = { self => true }
+        @kcode_map = {}
+      end
+
+      attr_reader :name, :kcode_map
+
+
+      def convertable_from(encoding)
+        @convertable_from[encoding] = true
+      end
+      protected :convertable_from
+
+      def changed
+        @convertable_from.each_key { |i| i.update_kcode_map }
+      end
+      private :changed
+
+
+      def kcode?
+        defined? @kcode
+      end
+
+      def kcode
+        if defined? @kcode then
+          @kcode
+        else
+          KCODE_None
+        end
+      end
+
+      def kcode=(kcode)
+        if defined? @kcode then
+          raise EncodingError, "KCODE conflict" unless @kcode == kcode
+        else
+          @kcode = kcode
+          changed
+        end
+        kcode
+      end
+
+
+      def converter(to)
+        @converter[to]
+      end
+
+      def add_converter(to, cost, conv_class)
+        if equal? to then
+          raise EncodingError,"attempt to add a converter to the same encoding"
+        end
+        oldconv = @converter[to]
+        if not oldconv or cost <= oldconv.cost then
+          conv = ConverterProperty.new(self, to, cost, conv_class)
+          @converter[to] = conv
+          to.convertable_from self
+          changed
+        end
+        nil
+      end
+
+
+      def update_kcode_map
+        @kcode_map.clear
+        @converter.each_value { |conv|
+          k = conv.to.kcode
+          if conv.to.kcode? and k then
+            oldconv = @kcode_map[k]
+            @kcode_map[k] = conv if not oldconv or conv.cost <= oldconv.cost
+          end
+        }
+      end
+      protected :update_kcode_map
+
+    end
+
+
+
+    def initialize
+      @encoding = {}
+    end
+
+    class << self
+      private :new
+      attr_reader :instance
+    end
+    @instance = new
+
+
+    private
+
+    def get_encoding(name)
+      encoding = @encoding[name.downcase]
+      raise EncodingError, "undeclared encoding `#{name}'" unless encoding
+      encoding
+    end
+
+    def touch_encoding(name)
+      name = name.downcase
+      encoding = @encoding[name]
+      encoding = @encoding[name] = EncodingProperty.new(name) unless encoding
+      encoding
+    end
+
+
+    public
+
+    def alias(newname, oldname)
+      newname = newname.downcase
+      if @encoding.key? newname then
+        raise EncodingError, "encoding `#{newname}' is already declared"
+      end
+      @encoding[newname] = get_encoding(oldname)
+      nil
+    end
+
+
+    def kcode(name)
+      encoding = @encoding[name.downcase]
+      if encoding then
+        encoding.kcode
+      else
+        KCODE_None
+      end
+    end
+
+
+    def set_kcode(name, kcode)
+      if kcode then
+        kcode = Regexp.new('', nil, kcode).kcode
+      else
+        kcode = nil
+      end
+      touch_encoding(name).kcode = kcode
+    end
+
+
+    def add_converter(from, to, cost, conv_class = nil, &block)
+      if block and conv_class then
+        raise ArgumentError, "multiple converters given"
+      elsif not block and not conv_class then
+        raise ArgumentError, "no converter given"
+      else
+        block = conv_class if Proc === conv_class
+        conv_class = SimpleConverter.new_class(block) if block
+      end
+      from = touch_encoding(from)
+      to = touch_encoding(to)
+      from.add_converter to, cost, conv_class
+    end
+
+
+    def converter(from, to)
+      fromenc = get_encoding(from)
+      toenc = get_encoding(to)
+      conv = fromenc.converter(toenc)
+      raise EncodingError, "can't convert `#{from}' to `#{to}'" unless conv
+      conv.new_converter
+    end
+
+
+    def converter3(from, to = nil)
+      to = from unless to
+      fromenc = get_encoding(from)
+      toenc = get_encoding(to)
+      kcode_map = fromenc.kcode_map
+      if kcode_map.empty? then
+        if fromenc.kcode and fromenc.equal? toenc then
+          return [ nil, fromenc.kcode, nil ]
+        else
+          raise EncodingError, "can't convert `#{from}' to any KCODE"
+        end
+      end
+      mincost, minkcode, minconv = nil
+      kcode_map.each { |kcode,conv|
+        conv2 = conv.to.converter(toenc)
+        if conv2 then
+          cost = conv.cost + conv2.cost
+          if not mincost or cost < mincost then
+            mincost, minkcode, minconv = cost, kcode, conv
+          end
+        end
+      }
+      unless mincost then
+        raise EncodingError, "can't convert `#{from}' to `#{to}' via any KCODE"
+      end
+      conv = minconv.new_converter
+      conv2 = minconv.to.converter(toenc)
+      conv2 = conv2 && conv2.new_converter
+      [ conv, minkcode, conv2 ]
+    end
+
+  end
+
+
+  Encoding = EncodingClass.instance
+
+  Encoding.set_kcode 'utf-8', 'U'
+  Encoding.set_kcode 'utf-16', nil
+  Encoding.alias 'iso-10646-ucs-2', 'utf-16'
+  Encoding.set_kcode 'iso-10646-ucs-4', nil
+  Encoding.set_kcode 'iso-8859-1', 'N'
+  Encoding.set_kcode 'iso-8859-2', 'N'
+  Encoding.set_kcode 'iso-8859-3', 'N'
+  Encoding.set_kcode 'iso-8859-4', 'N'
+  Encoding.set_kcode 'iso-8859-5', 'N'
+  Encoding.set_kcode 'iso-8859-6', 'N'
+  Encoding.set_kcode 'iso-8859-7', 'N'
+  Encoding.set_kcode 'iso-8859-8', 'N'
+  Encoding.set_kcode 'iso-8859-9', 'N'
+  Encoding.set_kcode 'iso-2022-jp', nil
+  Encoding.set_kcode 'shift_jis', 'S'
+  Encoding.set_kcode 'Windows-31J', 'S'
+  Encoding.set_kcode 'euc-jp', 'E'
+  Encoding.set_kcode 'euc-kr', 'E'
+
+end

data/redist/xmlscan/lib/xmlscan/htmlscan.rb

@@ -0,0 +1,289 @@
+#
+# xmlscan/htmlscan.rb
+#
+#   Copyright (C) Ueno Katsuhiro 2002
+#
+# $Id: htmlscan.rb,v 1.18 2003/05/01 15:36:50 katsu Exp $
+#
+
+require 'xmlscan/scanner'
+
+
+module XMLScan
+
+  class HTMLScanner < XMLScanner
+
+    private
+
+    def wellformed_error(msg)
+      # All wellformed error raised by XMLScanner are ignored.
+      # XMLScanner only raises wellformed error in stan_stag, which is a
+      # method completely overrided by HTMLScanner, so this method is
+      # never called in fact.
+    end
+
+    def on_xmldecl
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_version(str)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_encoding(str)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_standalone(str)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_other(name, value)
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_xmldecl_end
+      raise "[BUG] this method must be never called"
+    end
+
+    def on_stag_end_empty(name)
+      raise "[BUG] this method must be never called"
+    end
+
+
+    private
+
+    def scan_comment(s)
+      s[0,4] = ''  # remove `<!--'
+      comm = ''
+      until /--/n =~ s
+        comm << s
+        s = @src.get_plain
+        unless s then
+          parse_error "unterminated comment meets EOF"
+          return on_comment(comm)
+        end
+      end
+      comm << $`
+      s = $'
+      until s.empty? || s.strip.empty? and @src.close_tag   # --> or -- >
+        comm << '--'
+        if /\A\s*--/n =~ s then   # <!--hoge-- --
+          comm << $&
+          s = $'
+          if s.empty? and @src.close_tag then   # <!--hoge-- -->
+            parse_error "`-->' is found but comment must not end here"
+            comm.chop!.chop!
+            break
+          end
+        else                     # <!--hoge-- fuga
+          parse_error "only whitespace can appear between two comments"
+        end
+        if /\A-\s*\z/n =~ s and @src.close_tag then  # <!--hoge--->
+          parse_error "`-->' is found but comment must not end here"
+          comm.chop!
+          break
+        end
+        until /--/n =~ s      # copy & paste for performance
+          comm << s
+          s = @src.get_plain
+          unless s then
+            parse_error "unterminated comment meets EOF"
+            return on_comment(comm)
+          end
+        end
+        comm << $`
+        s = $'
+      end
+      on_comment comm
+    end
+
+
+    alias scan_xml_pi  scan_pi    # PIO "<?" PIC "?>"  -- <? PI ?> --
+
+
+    def scan_pi(s)   # <?PI >  this is default in SGML.
+      s[0,2] = ''    # remove `<?'
+      pi = s
+      until @src.close_tag
+        s = @src.get_plain
+        unless s then
+          parse_error "unterminated PI meets EOF"
+          break
+        end
+        pi << s
+      end
+      on_pi '', pi
+    end
+
+
+    def scan_stag(s)
+      unless /(?=[\/\s='"])/n =~ s then
+        name = s
+        name[0,1] = ''        # remove `<'
+        if name.empty? then   # <> or <<
+          if @src.close_tag then
+            return found_empty_stag
+          else
+            parse_error "parse error at `<'"
+            return on_chardata('<')
+          end
+        end
+        on_stag name
+        found_unclosed_stag name unless @src.close_tag
+        on_stag_end name
+      else
+        name = $`
+        s = $'
+        name[0,1] = ''        # remove `<'
+        if name.empty? then   # `< tag' or `<=`
+          parse_error "parse error at `<'"
+          if @src.close_tag then
+            s << '>'
+          end
+          return on_chardata('<' << s)
+        end
+        on_stag name
+        begin
+          continue = false
+          s.scan(
+         /([^\s=\/'"]+)(?:\s*=\s*(?:('[^']*'?|"[^"]*"?)|([^\s='"]+)))?|(\S)/n
+                 ) { |key,val,val2,error|
+            if key then
+              if val then                # key="value"
+                on_attribute key
+                qmark = val.slice!(0,1)
+                if val[-1] == qmark[0] then
+                  val.chop!
+                  scan_attvalue val unless val.empty?
+                else
+                  scan_attvalue val unless val.empty?
+                  begin
+                    s = @src.get
+                    unless s then
+                      parse_error "unterminated attribute `#{key}' meets EOF"
+                      break
+                    end
+                    c = s[0]
+                    val, s = s.split(qmark, 2)
+                    scan_attvalue '>' unless c == ?< or c == ?>
+                    scan_attvalue val if c
+                  end until s
+                  continue = s
+                end
+                on_attribute_end key
+              elsif val2 then            # key=value
+                on_attribute key
+                on_attr_value val2
+                on_attribute_end key
+              else                       # value
+                on_attribute nil
+                on_attr_value key
+                on_attribute_end nil
+              end
+            else
+              parse_error "parse error at `#{error}'"
+            end
+          }
+        end while continue
+        found_unclosed_stag name unless @src.close_tag
+        on_stag_end name
+      end
+    end
+
+
+    # This method should be called only from on_stag_end.
+    def get_cdata_content
+      unless not s = @src.test or s[0] == ?< && s[1] == ?/ then
+        dst = @src.get
+        until not s = @src.test or s[0] == ?< && s[1] == ?/
+          dst << @src.get_plain
+        end
+        dst
+      else
+        ''
+      end
+    end
+    public :get_cdata_content
+
+
+    def scan_bang_tag(s)
+      if s == '<!' and @src.close_tag then    # <!>
+        on_comment ''
+      else
+        parse_error "parse error at `<!'"
+        while s and not @src.close_tag        # skip entire
+          s = @src.get_plain
+        end
+      end
+    end
+
+
+    def scan_internal_dtd(s)
+      parse_error "DTD subset is found but it is not permitted in HTML"
+      skip_internal_dtd s
+    end
+
+
+    def found_invalid_pubsys(pubsys)
+      s = pubsys.upcase
+      return s if s == 'PUBLIC' or s == 'SYSTEM'
+      super
+    end
+
+
+    def scan_prolog(s)
+      doctype = 0
+      while s
+        if s[0] == ?< then
+          if (c = s[1]) == ?! then
+            if s[2] == ?- and s[3] == ?- then
+              scan_comment s
+            elsif /\A<!doctype(?=\s)/in =~ s then
+              doctype += 1
+              if doctype > 1 then
+                parse_error "another document type declaration is found"
+              end
+              scan_doctype $'
+            else
+              break
+            end
+          elsif c == ?? then
+            scan_pi s
+          else
+            break
+          end
+        elsif s.strip.empty? then
+          on_prolog_space s
+        else
+          break
+        end
+        s = @src.get
+      end
+      scan_content(s || @src.get)
+    end
+
+  end
+
+end
+
+
+
+
+
+if $0 == __FILE__ then
+  class TestVisitor
+    include XMLScan::Visitor
+    def parse_error(msg)
+      STDERR.printf("%s:%d: %s\n", $s.path, $s.lineno, msg) if $VERBOSE
+    end
+  end
+
+  $s = scan = XMLScan::HTMLScanner.new(TestVisitor.new)
+  src = ARGF
+  def src.path; filename; end
+  t1 = Time.times.utime
+  scan.parse src
+  t2 = Time.times.utime
+  STDERR.printf "%2.3f sec\n", t2 - t1
+end