xmlscan 0.2.3 → 0.3.0prea

data/README.processor

@@ -0,0 +1,33 @@
+
+  processor = XMLScan::Processor.hash(:element=>:card, :key=>:name) do |k,h,s,|
+    h[:transclude] || "{{#{h[:name]}}}"
+  end
+
+  test_cases [
+[ '<card name="foo" transclude="{{foo|titled}}">Some
+  <card name="name">Name data</card> and &lt; &gt;
+ <p>para data<b>bold</b>
+   </p><br/>
+ more<card
+   name="+hello" attr="&quote;foo&quote;"> and <card name="+nested">nested twice data</card>
+  </card>
+</card>
+', {
+'foo' => 'Some
+  {{name}} and &lt; &gt;
+ <p>para data<b>bold</b>
+   </p><br/>
+ more{{+hello}}
+',
+'name' => 'Name data',
+'foo+hello' => ' and {{+nested}}
+  ',
+'foo+hello+nested' => 'nested twice data' } ],
+  ]
+
+  test_casts.each { |p|
+    assert processor.call(p[0]) == p[1]
+  }
+
+
+

data/Rakefile

@@ -3,6 +3,9 @@
 
 require 'rubygems'
 require 'bundler'
+require 'xmlscan/version'
+
+VERSION = XMLScan::VERSION # File.exist?('VERSION') ? File.read('VERSION') : ""
 
 begin
   Bundler.setup(:default, :development)
@@ -15,10 +18,11 @@ end
 require 'rake'
 
 begin
+  include XMLScan
     require 'jeweler'
     Jeweler::Tasks.new do |gem|
-      gem.name = "xmlscan"
-      gem.version = '0.2.3'
+      gem.name = 'xmlscan'
+      gem.version = XMLScan::VERSION
       gem.license = "MIT"
       gem.summary = "The fastest XML parser written in 100% pure Ruby."
       gem.email = "gerryg@inbox.com"
@@ -56,10 +60,9 @@ task :default => :spec
 
 require 'rdoc/task'
 Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
 
   rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "xmlscan #{version}"
+  rdoc.title = "xmlscan #{VERSION}"
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.3.0prea

data/lib/xmlscan/htmlscan.rb

@@ -47,7 +47,7 @@ module XMLScan
       raise "[BUG] this method must be never called"
     end
 
-    def on_stag_end_empty(name)
+    def on_stag_end_empty(name, *a)
       raise "[BUG] this method must be never called"
     end
 
@@ -127,7 +127,7 @@ module XMLScan
             return found_empty_stag
           else
             parse_error "parse error at `<'"
-            return on_chardata('<')
+            return on_chardata '<'
           end
         end
         on_stag name
@@ -142,7 +142,7 @@ module XMLScan
           if @src.close_tag then
             s << '>'
           end
-          return on_chardata('<' << s)
+          return on_chardata '<'+s
         end
         on_stag name
         begin
@@ -156,9 +156,9 @@ module XMLScan
                 qmark = val.slice!(0,1)
                 if val[-1] == qmark[0] then
                   val.chop!
-                  scan_attvalue val unless val.empty?
+                  scan_attr_value val unless val.empty?
                 else
-                  scan_attvalue val unless val.empty?
+                  scan_attr_value val unless val.empty?
                   begin
                     s = @src.get
                     unless s then
@@ -167,8 +167,8 @@ module XMLScan
                     end
                     c = s[0]
                     val, s = s.split(qmark, 2)
-                    scan_attvalue '>' unless c == ?< or c == ?>
-                    scan_attvalue val if c
+                    scan_attr_value '>' unless c == ?< or c == ?>
+                    scan_attr_value val if c
                   end until s
                   continue = s
                 end

data/lib/xmlscan/namespace.rb

@@ -54,16 +54,16 @@ module XMLScan
     #       on_stag_end_empty_ns ('foo:bar', { 'foo' => '', ... })
     #
 
-    def on_stag_ns(qname, prefix, localpart)
+    def on_stag_ns(qname, prefix, localpart, *a)
     end
 
-    def on_attribute_ns(qname, prefix, localpart)
+    def on_attribute_ns(qname, prefix, localpart, *a)
     end
 
-    def on_stag_end_ns(qname, namespaces)
+    def on_stag_end_ns(qname, namespaces, *a)
     end
 
-    def on_stag_end_empty_ns(qname, namespaces)
+    def on_stag_end_empty_ns(qname, namespaces, *a)
     end
 
   end
@@ -99,7 +99,7 @@ module XMLScan
     end
 
 
-    def on_start_document
+    def on_start_document(*a)
       @namespace = {} #PredefinedNamespace.dup
       @ns_hist = []
       @ns_undeclared = {}     # for checking undeclared namespace prefixes.
@@ -107,14 +107,14 @@ module XMLScan
       @dont_same = []         # ditto.
       @xmlns = NamespaceDeclaration.new(self)
       @orig_visitor = @visitor
-      @visitor.on_start_document
+      @visitor.on_start_document *a
     end
 
 
-    def on_stag(name)
+    def on_stag(name, *a)
       @ns_hist.push nil
       unless /:/n =~ name then
-        @visitor.on_stag_ns name, '', name
+        @visitor.on_stag_ns name, '', name, *a
       else
         prefix, localpart = $`, $'
         if localpart.include? ?: then
@@ -131,12 +131,12 @@ module XMLScan
             @ns_undeclared[prefix] = true
           end
         end
-        @visitor.on_stag_ns name, prefix, localpart
+        @visitor.on_stag_ns name, prefix, localpart, *a
       end
     end
 
 
-    def on_attribute(name)
+    def on_attribute(name, *a)
       if /:/n =~ name then
         prefix, localpart = $`, $'
         if localpart.include? ?: then
@@ -157,13 +157,13 @@ module XMLScan
             @dont_same.push [ prev, prefix, localpart ]
           end
           @prev_prefix[localpart] = prefix
-          @visitor.on_attribute_ns name, prefix, localpart
+          @visitor.on_attribute_ns name, prefix, localpart, *a
         end
       elsif name == 'xmlns' then
         @visitor = @xmlns
         @xmlns.on_xmlns_start ''
       else
-        @visitor.on_attribute_ns name, nil, name
+        @visitor.on_attribute_ns name, nil, name, *a
       end
     end
 
@@ -176,36 +176,36 @@ module XMLScan
         @parent = parent
       end
 
-      def on_xmlns_start(prefix)
+      def on_xmlns_start(prefix, *a)
         @prefix = prefix
         @nsdecl = ''
       end
 
-      def on_attr_value(str)
+      def on_attr_value(str, *a)
         @nsdecl << str
       end
 
-      def on_attr_entityref(ref)
+      def on_attr_entityref(ref, *a)
         @parent.ns_wellformed_error \
           "xmlns includes undeclared entity reference"
       end
 
-      def on_attr_charref(code)
+      def on_attr_charref(code, *a)
         @nsdecl << [code].pack('U')
       end
 
-      def on_attr_charref_hex(code)
+      def on_attr_charref_hex(code, *a)
         @nsdecl << [code].pack('U')
       end
 
-      def on_attribute_end(name)
+      def on_attribute_end(name, *a)
         @parent.on_xmlns_end @prefix, @nsdecl
       end
 
     end
 
 
-    def on_xmlns_end(prefix, uri)
+    def on_xmlns_end(prefix, uri, *a)
       @visitor = @orig_visitor
       if PredefinedNamespace.key? prefix then
         if prefix == 'xmlns' then
@@ -254,54 +254,54 @@ module XMLScan
     end
 
 
-    def on_stag_end(name)
+    def on_stag_end(name, *a)
       fix_namespace
-      @visitor.on_stag_end_ns name, @namespace
+      @visitor.on_stag_end_ns name, @namespace, *a
     end
 
 
-    def on_etag(name)
+    def on_etag(name, *a)
       h = @ns_hist.pop and @namespace.update h
-      @visitor.on_etag name
+      @visitor.on_etag name, *a
     end
 
 
-    def on_stag_end_empty(name)
+    def on_stag_end_empty(name, *a)
       fix_namespace
-      @visitor.on_stag_end_empty_ns name, @namespace
+      @visitor.on_stag_end_empty_ns name, @namespace, *a
       h = @ns_hist.pop and @namespace.update h
     end
 
 
-    def on_doctype(root, pubid, sysid)
+    def on_doctype(root, pubid, sysid, *a)
       if root.count(':') > 1 then
         ns_parse_error "qualified name `#{root}' includes `:'"
       end
-      @visitor.on_doctype root, pubid, sysid
+      @visitor.on_doctype root, pubid, sysid, *a
     end
 
 
-    def on_pi(target, pi)
+    def on_pi(target, pi, *a)
       if target.include? ?: then
         ns_parse_error "PI target `#{target}' includes `:'"
       end
-      @visitor.on_pi target, pi
+      @visitor.on_pi target, pi, *a
     end
 
 
-    def on_entityref(ref)
+    def on_entityref(ref, *a)
       if ref.include? ?: then
         ns_parse_error "entity reference `#{ref}' includes `:'"
       end
-      @visitor.on_entityref ref
+      @visitor.on_entityref ref, *a
     end
 
 
-    def on_attr_entityref(ref)
+    def on_attr_entityref(ref, *a)
       if ref.include? ?: then
         ns_parse_error "entity reference `#{ref}' includes `:'"
       end
-      @visitor.on_attr_entityref ref
+      @visitor.on_attr_entityref ref, *a
     end
 
   end

data/lib/xmlscan/parser.rb

@@ -43,7 +43,7 @@ module XMLScan
 
     private
 
-    def on_xmldecl_version(str)
+    def on_xmldecl_version(str, *a)
       unless str == '1.0' then
         warning "unsupported XML version `#{str}'"
       end
@@ -51,7 +51,7 @@ module XMLScan
     end
 
 
-    def on_xmldecl_standalone(str)
+    def on_xmldecl_standalone(str, *a)
       if str == 'yes' then
         @standalone = true
       elsif str == 'no' then
@@ -63,7 +63,7 @@ module XMLScan
     end
 
 
-    def on_doctype(name, pubid, sysid)
+    def on_doctype(name, pubid, sysid, *a)
       if pubid and not sysid then
         parse_error "public external ID must have both public ID and system ID"
       end
@@ -71,12 +71,12 @@ module XMLScan
     end
 
 
-    def on_prolog_space(s)
+    def on_prolog_space(s, *a)
       # just ignore it.
     end
 
 
-    def on_pi(target, pi)
+    def on_pi(target, pi, *a)
       if target.downcase == 'xml' then
         parse_error "reserved PI target `#{target}'"
       end
@@ -114,39 +114,43 @@ module XMLScan
     #end
 
 
-    def on_stag(name)
+    def on_stag(name, *a)
       @elem.push name
       @visitor.on_stag name
       @attr.clear
     end
 
-    def on_attribute(name)
+    def on_attribute(name, *a)
       unless @attr.check_unique name then
         wellformed_error "doubled attribute `#{name}'"
       end
       @visitor.on_attribute name
     end
 
-    def on_attr_value(str)
+    def on_attr_value(str, *a)
       str.tr! "\t\r\n", ' '   # normalize
       @visitor.on_attr_value str
     end
 
-    def on_stag_end_empty(name)
+    def on_stag_end(name, *a)
+      @visitor.on_stag_end name, *a
+    end
+
+    def on_stag_end_empty(name, *a)
       # @visitor.on_stag_end name
       # @elem.pop
       # @visitor.on_etag name
-      @visitor.on_stag_end_empty name
+      @visitor.on_stag_end_empty name, *a
       @elem.pop
     end
 
-    def on_etag(name)
+    def on_etag(name, *a)
       last = @elem.pop
       if last == name then
-        @visitor.on_etag name
+        @visitor.on_etag name, *a
       elsif last then
         wellformed_error "element type `#{name}' is not matched"
-        @visitor.on_etag last
+        @visitor.on_etag last, *a
       else
         parse_error "end tag `#{name}' appears alone"
       end

data/lib/xmlscan/processor.rb

@@ -0,0 +1,47 @@
+# encoding: UTF-8
+require 'xmlscan/parser'
+require 'xmlscan/visitor'
+
+module XMLScan
+  module ElementProcessor
+    include XMLScan::Visitor
+
+    SKIP = [:on_chardata, :on_stag, :on_etag, :on_attribute, :on_attr_entityref,
+      :on_attr_value, :on_start_document, :on_end_document, :on_attribute_end,
+      :on_stag_end, :on_stag_end_empty, :on_attr_charref, :on_attr_charref_hex]
+
+    MY_METHODS = XMLScan::Visitor.instance_methods.to_a - SKIP
+
+    def initialize(opts={}, mod=nil)
+      (mod ? MY_METHODS - mod.instance_methods : MY_METHODS).each do |i|
+        self.class.class_eval %{def #{i}(d, *a) d&&(@out << d) end}, __FILE__, __LINE__
+      end
+      self.class.send :include, mod
+
+      @element = opts[:element] || raise("need an element")
+      @key = opts[:key] || raise("need a key")
+      @extras = (ex = opts[:extras]) ? ex.map(&:to_sym) : []
+
+      @pairs = []    # output [name, content, value] * 1 or more
+      @context = ''  # current key(name) of the element (card)
+      @stack = []    # stack of containing context cards
+      @out = []      # current output for name(card)
+      @parser = XMLScan::XMLParser.new(self)
+      self
+    end
+
+  end
+
+  class XMLProcessor
+    include ElementProcessor
+
+    def self.process(file, opts={}, mod=nil)
+      raise "Not readable #{file.inspect}" unless IO===( io =
+                                             IO===file ? file : open(file) )
+      visitor = new(opts, mod)
+      visitor.parser.parse(io)
+      visitor.pairs
+    end
+  end
+
+end

data/lib/xmlscan/scanner.rb

@@ -122,20 +122,29 @@ module XMLScan
       self
     end
 
-
+=begin
+  Managing source in a private array.
+  * tag oriented (?< and ?> are the key tokens
+  * ?> that aren't followed by another ?< or ?> are stripped in splitting
+=end
     def get
       pop or
         unless @eof then
           last = @last
           begin
-            src = @src.gets
-            unless src then
+            unless chunk = @src.gets then
               @eof = true
-              unshift last
-              last = nil
-              break
+              @last = nil
+              return last
+              #unshift last # to be popped after reverse!
+              #last = nil
+              #break
             end
-            a = src.split(/(?=<|>[<>])|>/, -1)
+            # negative lookahead: < or >< or >>
+            # so don't consume those (but split leaving them always at the
+            # end of chunks)
+            # consume (>) and split on >
+            a = chunk.split(/(?=<|>[<>])|>/, -1)
             if last then
               unless /\A[<>]/ =~ a.first then
                 a[0] = last << (a.first || '')
@@ -143,6 +152,7 @@ module XMLScan
                 push last
               end
             end
+            raise "size #{size}" if size > 1
             concat a
             last = pop
           end while empty?
@@ -223,7 +233,7 @@ module XMLScan
         last.push @last.inspect
       end
       a.push '#eof' if @eof
-      "((#{a.join(' ')}) (#{last.join(' ')}) . #{source.inspect})"
+      "((#{a*' '}) l(#{last*' '}) . #{source.inspect})"
     end
 
     def each
@@ -354,72 +364,72 @@ module XMLScan
       end
     end
 
-    def on_xmldecl_version(str)
-      @visitor.on_xmldecl_version str
+    def on_xmldecl_version(str, *a)
+      @visitor.on_xmldecl_version str, *a
     end
 
-    def on_xmldecl_encoding(str)
-      @visitor.on_xmldecl_encoding str
+    def on_xmldecl_encoding(str, *a)
+      @visitor.on_xmldecl_encoding str, *a
     end
 
-    def on_xmldecl_standalone(str)
-      @visitor.on_xmldecl_standalone str
+    def on_xmldecl_standalone(str, *a)
+      @visitor.on_xmldecl_standalone str, *a
     end
 
-    def on_xmldecl_other(name, value)
-      @visitor.on_xmldecl_other name, value
+    def on_xmldecl_other(name, value, *a)
+      @visitor.on_xmldecl_other name, value, *a
     end
 
-    def on_xmldecl_end
-      @visitor.on_xmldecl_end
+    def on_xmldecl_end(*a)
+      @visitor.on_xmldecl_end *a
     end
 
-    def on_doctype(root, pubid, sysid)
-      @visitor.on_doctype root, pubid, sysid
+    def on_doctype(root, pubid, sysid, *a)
+      @visitor.on_doctype root, pubid, sysid, *a
     end
 
-    def on_prolog_space(str)
-      @visitor.on_prolog_space str
+    def on_prolog_space(str, *a)
+      @visitor.on_prolog_space str, *a
     end
 
-    def on_comment(str)
-      @visitor.on_comment str
+    def on_comment(str, *a)
+      @visitor.on_comment str, *a
     end
 
-    def on_pi(target, pi)
-      @visitor.on_pi target, pi
+    def on_pi(target, pi, *a)
+      @visitor.on_pi target, pi, *a
     end
 
-    def on_chardata(str)
-      @visitor.on_chardata str
+    def on_chardata(str, *a)
+      @visitor.on_chardata str, *a
     end
 
-    def on_cdata(str)
-      @visitor.on_cdata str
+    def on_cdata(str, *a)
+      @visitor.on_cdata str, *a
     end
 
-    def on_etag(name)
-      @visitor.on_etag name
+    def on_etag(name, *a)
+      @visitor.on_etag name, *a
     end
 
-    def on_entityref(ref)
-      @visitor.on_entityref ref
+    def on_entityref(ref, *a)
+      @visitor.on_entityref ref, *a
     end
 
-    def on_charref(code)
-      @visitor.on_charref code
+    def on_charref(code, *a)
+      @visitor.on_charref code, *a
     end
 
-    def on_charref_hex(code)
-      @visitor.on_charref_hex code
+    def on_charref_hex(code, *a)
+      @visitor.on_charref_hex code, *a
     end
 
-    def on_start_document
-      @visitor.on_start_document
+    def on_start_document(*a)
+      @visitor.on_start_document *a
     end
 
-    def on_end_document
-      @visitor.on_end_document
+    def on_end_document(*a)
+      @visitor.on_end_document *a
     end
 
 
@@ -444,50 +454,51 @@ module XMLScan
     #
     #    A: on_chardata          ('HOGE')
 
-    def on_stag(name)
-      @visitor.on_stag name
+    def on_stag(name, *a)
+      @visitor.on_stag name, *a
     end
 
-    def on_attribute(name)
-      @visitor.on_attribute name
+    def on_attribute(name, *a)
+      @visitor.on_attribute name, *a
     end
 
-    def on_attr_value(str)
-      @visitor.on_attr_value str
+    def on_attr_value(str, *a)
+      @visitor.on_attr_value str, *a
     end
 
-    def on_attr_entityref(ref)
-      @visitor.on_attr_entityref ref
+    def on_attr_entityref(ref, *a)
+      @visitor.on_attr_entityref ref, *a
     end
 
-    def on_attr_charref(code)
-      @visitor.on_attr_charref code
+    def on_attr_charref(code, *a)
+      @visitor.on_attr_charref code, *a
     end
 
-    def on_attr_charref_hex(code)
-      @visitor.on_attr_charref_hex code
+    def on_attr_charref_hex(code, *a)
+      @visitor.on_attr_charref_hex code, *a
     end
 
-    def on_attribute_end(name)
-      @visitor.on_attribute_end name
+    def on_attribute_end(name, *a)
+      @visitor.on_attribute_end name, *a, *a
     end
 
-    def on_stag_end_empty(name)
-      @visitor.on_stag_end_empty name
+    def on_stag_end_empty(name, *a)
+      @visitor.on_stag_end_empty name, *a
     end
 
-    def on_stag_end(name)
-      @visitor.on_stag_end name
+    def on_stag_end(name, *a)
+      #STDERR << "ose #{name}, #{a.inspect}\n"
+      @visitor.on_stag_end name, *a
     end
 
 
+    S_OPT_EXAMPLE = "".encode(::Encoding::WINDOWS_31J)
+    E_OPT_EXAMPLE = "".encode(::Encoding::EUCJP)
 
     private
 
     module OptRegexp
       UTFSTR = "é"
-      S_OPT_EXAMPLE = "".encode Encoding.find('Windows-31J')
-      E_OPT_EXAMPLE = "".encode Encoding.find('EUC-JP')
 
       RE_ENCODINGS = {
         :n=>/e/n.encoding,
@@ -525,6 +536,7 @@ module XMLScan
         else
           s = $`
           on_chardata s unless s.empty?
+          #orig = $'.sub(/(?=;).*$/,'')
           ref = nil
           $'.split('&', -1).each { |s|
             unless /(?!\A);|(?=[ \t\r\n])/ =~ s and not $&.empty? then
@@ -533,18 +545,18 @@ module XMLScan
                 parse_error "reference to `#{ref}' doesn't end with `;'"
               else
                 parse_error "`&' is not used for entity/character references"
-                on_chardata('&' << s)
+                on_chardata '&'+s
                 next
               end
             end
-            ref = $`
+            orig = ?& + (ref = $`) + ?;
             s = $'
             if /\A[^#]/ =~ ref then
-              on_entityref ref
+              on_entityref ref, orig
             elsif /\A#(\d+)\z/ =~ ref then
-              on_charref $1.to_i
+              on_charref $1.to_i, orig
             elsif /\A#x([\dA-Fa-f]+)\z/ =~ ref then
-              on_charref_hex $1.hex
+              on_charref_hex $1.hex, orig
             else
               parse_error "invalid character reference `#{ref}'"
             end
@@ -558,8 +570,9 @@ module XMLScan
     end
 
 
-    def scan_attvalue(s)     # almostly copy & paste from scan_chardata
+    def scan_attr_value(s)     # almostly copy & paste from scan_chardata
       unless /&/ =~ s then
+        #STDERR << "no& attr_val #{s.inspect}, #{caller*"\n"}\n" if s == ?>
         on_attr_value s
       else
         s = $`
@@ -576,14 +589,14 @@ module XMLScan
               next
             end
           end
-          ref = $`
+          orig = ?& + (ref = $`) + ?;
           s = $'
           if /\A[^#]/ =~ ref then
-            on_attr_entityref ref
+            on_attr_entityref ref, orig
           elsif /\A#(\d+)\z/ =~ ref then
-            on_attr_charref $1.to_i
+            on_attr_charref $1.to_i, orig
           elsif /\A#x([\dA-Fa-f]+)\z/ =~ ref then
-            on_attr_charref_hex $1.hex
+            on_attr_charref_hex $1.hex, orig
           else
             parse_error "invalid character reference `#{ref}'"
           end
@@ -682,6 +695,7 @@ module XMLScan
 
 
     def scan_etag(s)
+      orig="#{s}>"
       s[0,2] = ''  # remove '</'
       if s.empty? then
         if @src.close_tag then   # </>
@@ -689,14 +703,14 @@ module XMLScan
         else                     # </< or </[EOF]
           parse_error "parse error at `</'"
           s << '>' if @src.close_tag
-          return on_chardata('</' << s)
+          return on_chardata '</' << s
         end
       elsif /[ \t\n\r]+/ =~ s then
         s1, s2 = $`, $'
         if s1.empty? then                # </ tag
           parse_error "parse error at `</'"
           s << '>' if @src.close_tag
-          return on_chardata('</' + s)
+          return on_chardata '</' + s
         elsif not s2.empty? then         # </ta g
           parse_error "illegal whitespace is found within end tag `#{s1}'"
           while @src.get_tag
@@ -705,7 +719,7 @@ module XMLScan
         s = s1
       end
       found_unclosed_etag s unless @src.close_tag   # </tag< or </tag[EOF]
-      on_etag s
+      on_etag s, orig
     end
 
 
@@ -745,6 +759,8 @@ module XMLScan
 
 
     def scan_stag(s)
+      hash = {}
+      orig = [s.dup] 
       unless /(?=[\/ \t\n\r='"])/ =~ s then
         name = s
         name[0,1] = ''        # remove `<'
@@ -753,54 +769,65 @@ module XMLScan
             return found_empty_stag
           else                     # << or <[EOF]
             parse_error "parse error at `<'"
-            return on_chardata('<')
+            return on_chardata '<'
           end
         end
         on_stag name
         found_unclosed_stag name unless @src.close_tag
-        on_stag_end name
+        on_stag_end name, orig*''+?>, {}
       else
+        k = nil
         name = $`
         s = $'
         name[0,1] = ''        # remove `<'
         if name.empty? then   # `< tag' or `<=`
           parse_error "parse error at `<'"
           s << '>' if @src.close_tag
-          return on_chardata('<' << s)
+          return on_chardata '<' << s
         end
         on_stag name
         emptyelem = false
-        key,val,error,qmark,c = nil
         begin
           continue = false
           s.scan(/[ \t\n\r]([^= \t\n\r\/'"]+)[ \t\n\r]*=[ \t\n\r]*('[^']*'?|"[^"]*"?)|\/\z|([^ \t\n\r][\S\s]*)/
                  ) { |key,val,error|
-            if key then                # key="value"
+            orig_val = []
+            if key then
               on_attribute key
+              k=key
+              orig_val << val
               qmark = val.slice!(0,1)
               if val[-1] == qmark[0] then
                 val.chop!
-                scan_attvalue val unless val.empty?
+                scan_attr_value val unless val.empty?
               else
-                scan_attvalue val unless val.empty?
+                scan_attr_value val unless val.empty?
                 begin
                   s = @src.get
+                  #STDERR << "get some more? #{s.inspect}, #{orig.inspect}\n"
                   unless s then
                     parse_error "unterminated attribute `#{key}' meets EOF"
                     break
                   end
+                  orig << s.dup
                   c = s[0]
                   val, s = s.split(qmark, 2)
+                  orig_val << val
                   if c == ?< then
                     wellformed_error "`<' is found in attribute `#{key}'"
                   elsif c != ?> then
-                    scan_attvalue '>'
+                    #STDERR << "close in quote? #{c.inspect}, #{@src.tag_start?}, #{@src.tag_end?}, #{s.inspect}, #{val.inspect}, #{orig.inspect}, #{orig_val.inspect}\n"
+                    orig_val[-1,0] = orig[-1,0] = ?> # if @src.tag_start?
+                    scan_attr_value ?>
                   end
-                  scan_attvalue val if c
+                  scan_attr_value val if c
                 end until s
                 continue = s      # if eof then continue is false, else true.
               end
-              on_attribute_end key
+              #STDERR << "attr:#{k}, #{orig_val}\n"
+              hash[k] = orig_val*''
+              #STDERR << "attr end #{hash.inspect}, #{k}, #{orig_val}\n"
+              on_attribute_end key #, orig_val*''
             elsif error then
               continue = s = found_stag_error(error)
             else
@@ -816,9 +843,11 @@ module XMLScan
           end
         end
         if emptyelem then
-          on_stag_end_empty name
+          on_stag_end_empty name, orig*''+?>, hash
         else
-          on_stag_end name
+          #STDERR << "on stag end #{ name}, \"<#{name}#{s}>\", #{hash.inspect}\n"
+          on_stag_end name, orig*''+?>, hash
+          #on_stag_end name, "<#{name}#{s}>", hash
         end
       end
     end
@@ -1067,10 +1096,10 @@ module XMLScan
 
 
     def scan_document
-      on_start_document
+      on_start_document ''
       @src.prepare
       scan_prolog @src.get
-      on_end_document
+      on_end_document ''
     end
 
 

data/lib/xmlscan/version.rb

@@ -9,15 +9,9 @@
 
 module XMLScan
 
-  # The version like 'X.X.0' (TENNY is 0) means that this is an unstable
-  # release. Incompatible changes will be applied to this version
-  # without special notice. This version should be distributed as a
-  # snapshot only.
-  #
-  # TENNY which is larger than 1 (e.g. 'X.X.1' or 'X.X.2') means this
-  # release is a stable release.
-
-  VERSION = '0.2.3'
-  RELEASE_DATE = '2003-05-02'
+  GEMNAME = 'xmlscan'
+  VERSION_FILE = File.expand_path('../../VERSION', File.dirname(__FILE__))
+  VERSION = open(VERSION_FILE).to_a*''.chop
+  RELEASE_DATE = open(VERSION_FILE).mtime.strftime('%Y-%m-%d')
 
 end

data/lib/xmlscan/visitor.rb

@@ -54,88 +54,88 @@ module XMLScan
     def warning(msg)
     end
 
-    def on_xmldecl
+    def on_xmldecl(*a)
     end
 
-    def on_xmldecl_key(key, str)
+    def on_xmldecl_key(key, str, *a)
     end
 
-    def on_xmldecl_version(str)
+    def on_xmldecl_version(str, *a)
     end
 
-    def on_xmldecl_encoding(str)
+    def on_xmldecl_encoding(str, *a)
     end
 
-    def on_xmldecl_standalone(str)
+    def on_xmldecl_standalone(str, *a)
     end
 
-    def on_xmldecl_other(name, value)
+    def on_xmldecl_other(name, value, *a)
     end
 
-    def on_xmldecl_end
+    def on_xmldecl_end(*a)
     end
 
-    def on_doctype(root, pubid, sysid)
+    def on_doctype(root, pubid, sysid, *a)
     end
 
-    def on_prolog_space(str)
+    def on_prolog_space(str, *a)
     end
 
-    def on_comment(str)
+    def on_comment(str, *a)
     end
 
-    def on_pi(target, pi)
+    def on_pi(target, pi, *a)
     end
 
-    def on_chardata(str)
+    def on_chardata(str, *a)
     end
 
-    def on_cdata(str)
+    def on_cdata(str, *a)
     end
 
-    def on_etag(name)
+    def on_etag(name, *a)
     end
 
-    def on_entityref(ref)
+    def on_entityref(ref, *a)
     end
 
-    def on_charref(code)
+    def on_charref(code, *a)
     end
 
-    def on_charref_hex(code)
+    def on_charref_hex(code, *a)
     end
 
-    def on_start_document
+    def on_start_document(*a)
     end
 
-    def on_end_document
+    def on_end_document(*a)
     end
 
-    def on_stag(name)
+    def on_stag(name, *a)
     end
 
-    def on_attribute(name)
+    def on_attribute(name, *a)
     end
 
-    def on_attr_value(str)
+    def on_attr_value(str, *a)
     end
 
-    def on_attr_entityref(ref)
+    def on_attr_entityref(ref, *a)
     end
 
-    def on_attr_charref(code)
+    def on_attr_charref(code, *a)
     end
 
-    def on_attr_charref_hex(code)
+    def on_attr_charref_hex(code, *a)
     end
 
-    def on_attribute_end(name)
+    def on_attribute_end(name, *a)
     end
 
-    def on_stag_end_empty(name)
+    def on_stag_end_empty(name, *a)
     end
 
-    def on_stag_end(name)
+    def on_stag_end(name, *a)
     end
 
   end
@@ -146,13 +146,15 @@ module XMLScan
     include Visitor
 
     def initialize(visitor)
+      #STDERR << "new Decoration #{visitor}\n"
       @visitor = visitor
     end
 
     Visitor.instance_methods.each { |i|
+          #STDERR << "#{i} \#{args.inspect}\\n"
       module_eval <<-END, __FILE__, __LINE__ + 1
         def #{i}(*args)
-          @visitor.#{i}(*args)
+          @visitor&&@visitor.#{i}(*args)
         end
       END
     }

data/lib/xmlscan/xmlchar.rb

@@ -115,95 +115,95 @@ module XMLScan
       end
 
 
-      def on_xmldecl_version(str)
+      def on_xmldecl_version(str, *a)
         check_valid_version str
         super
       end
 
-      def on_xmldecl_encoding(str)
+      def on_xmldecl_encoding(str, *a)
         check_valid_encoding str
         super
       end
 
-      def on_xmldecl_standalone(str)
+      def on_xmldecl_standalone(str, *a)
         check_valid_chardata str
         super
       end
 
-      def on_doctype(root, pubid, sysid)
+      def on_doctype(root, pubid, sysid, *a)
         check_valid_name root
         check_valid_pubid pubid if pubid
         check_valid_chardata sysid if sysid
         super
       end
 
-      def on_comment(str)
+      def on_comment(str, *a)
         check_valid_chardata str
         super
       end
 
-      def on_pi(target, pi)
+      def on_pi(target, pi, *a)
         check_valid_name target
         check_valid_chardata pi
         super
       end
 
-      def on_chardata(str)
+      def on_chardata(str, *a)
         check_valid_chardata str
         super
       end
 
-      def on_cdata(str)
+      def on_cdata(str, *a)
         check_valid_chardata str
         super
       end
 
-      def on_etag(name)
+      def on_etag(name, *a)
         check_valid_name name
         super
       end
 
-      def on_entityref(ref)
+      def on_entityref(ref, *a)
         check_valid_name ref
         super
       end
 
-      def on_charref(code)
+      def on_charref(code, *a)
         check_valid_char code
         super
       end
 
-      def on_charref_hex(code)
+      def on_charref_hex(code, *a)
         check_valid_char code
         super
       end
 
-      def on_stag(name)
+      def on_stag(name, *a)
         check_valid_name name
         super
       end
 
-      def on_attribute(name)
+      def on_attribute(name, *a)
         check_valid_name name
         super
       end
 
-      def on_attr_value(str)
+      def on_attr_value(str, *a)
         check_valid_chardata str
         super
       end
 
-      def on_attr_entityref(ref)
+      def on_attr_entityref(ref, *a)
         check_valid_name ref
         super
       end
 
-      def on_attr_charref(code)
+      def on_attr_charref(code, *a)
         check_valid_char code
         super
       end
 
-      def on_attr_charref_hex(code)
+      def on_attr_charref_hex(code, *a)
         check_valid_char code
         super
       end

data/xmlcard.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+
+require 'xmlscan/processor'
+
+# need to make these into supplied blocks somehome
+module CustomProcessing
+  def on_chardata(s) @out << s end
+  def on_stag_end(name, s, h, *a)
+    if name.to_sym == @element
+      # starting a new context, first output our substitute string
+      key= h&&h[@key.to_s]||'*no-name*'
+      sub = h['transclude'] || "{{#{key}}}"
+      @out << sub
+      # then push the current context and initialize this one
+      @stack.push([@context, @out, *@ex])
+      @context = key; @out = []
+      @ex = @extras.map {|e| h[e]}
+    else @out << s end # pass through tags we aren't processing
+  end
+
+  def on_etag(name, s=nil)
+    if name.to_sym == @element
+      # output a card (name, content, type)
+      @pairs << [@context, @out, @stack[-1][0], *@ex]
+      # restore previous context from stack
+      last = @stack.pop
+      @context, @out, @ex = last.shift, last.shift, *last
+    else @out << s end
+  end
+
+  def on_stag_empty_end(name, s=nil, h={}, *a)
+    if name.to_sym == @element
+      # I don't think we have this case, but it is simple to add later
+      STDERR << "empty card ???: #{name}, #{s}, #{h.inspect}\n"
+    else @out << s end
+  end
+
+  attr_reader :pairs, :parser
+end
+
+ARGV.each do |a|
+  pairs = XMLScan::XMLProcessor.process(a, {:key=>:name, :element=>:card, :extras=>[:type]}, CustomProcessing)
+  STDOUT << "Result\n"
+  STDOUT << pairs.map do |p| n,o,c,t = p
+      "#{c&&c.size>0&&"#{c}::"||''}#{n}#{t&&"[#{t}]"}=>#{o*''}"
+    end * "\n"
+  STDOUT << "\nDone\n"
+end

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: xmlscan
 version: !ruby/object:Gem::Version
-  version: 0.2.3
-  prerelease: 
+  version: 0.3.0prea
+  prerelease: 5
 platform: ruby
 authors:
 - UENO Katsuhiro <katsu@blue.sky.or.jp>
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-13 00:00:00.000000000 Z
+date: 2012-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &8077620 !ruby/object:Gem::Requirement
+  requirement: &9220620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 2.8.0
   type: :development
   prerelease: false
-  version_requirements: *8077620
+  version_requirements: *9220620
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &8076660 !ruby/object:Gem::Requirement
+  requirement: &9220120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *8076660
+  version_requirements: *9220120
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &8075620 !ruby/object:Gem::Requirement
+  requirement: &9219620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *8075620
+  version_requirements: *9219620
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &8074720 !ruby/object:Gem::Requirement
+  requirement: &9219060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,7 +54,7 @@ dependencies:
         version: 1.8.3
   type: :development
   prerelease: false
-  version_requirements: *8074720
+  version_requirements: *9219060
 description: The fastest XML parser written in 100% pure Ruby.
 email: gerryg@inbox.com
 executables: []
@@ -65,6 +65,7 @@ files:
 - ChangeLog
 - Gemfile
 - Gemfile.lock
+- README.processor
 - README.rdoc
 - Rakefile
 - THANKS
@@ -73,11 +74,13 @@ files:
 - lib/xmlscan/htmlscan.rb
 - lib/xmlscan/namespace.rb
 - lib/xmlscan/parser.rb
+- lib/xmlscan/processor.rb
 - lib/xmlscan/scanner.rb
 - lib/xmlscan/version.rb
 - lib/xmlscan/visitor.rb
 - lib/xmlscan/xmlchar.rb
 - test.rb
+- xmlcard.rb
 homepage: http://github.com/GerryG/xmlformat/
 licenses:
 - MIT
@@ -97,13 +100,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3268123461909302440
+      hash: -1617079850723236327
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.15