rgen 0.3.0 → 0.4.0

data/redist/xmlscan/memo/PRODUCTIONS

@@ -0,0 +1,195 @@
+$Id: PRODUCTIONS,v 1.3 2003/01/19 14:35:49 katsu Exp $
+
+== $B<BAu$N40N;$7$F$$$k$b$N!#(B
+
+$B$3$l$i$N@8@.5,B'$K%^%C%A$9$k$3$H$,4|BT$5$l$kJ8L.$K$*$$$F!"4|BT$5$l$k@8@.5,B'$K(B
+$B%^%C%A$7$J$+$C$?>l9g$O!">o$K(B parse error $B$H$J$k!#(B
+
+
+XMLScan::XMLScanner $B$G<BAu$5$l$F$$$k$b$N!#(B
+
+ [10]           AttValue ::= '"' ([^<&"] | Reference)* '"'
+                             | ("'" ([^<&'] | Reference)* "'")
+ [14]           CharData ::= [^<&]* - ([^<&]* ']]>' [^<&]*)
+ [15]            Comment ::= '<!--' ((Char - '-') | ('-' (Char - '-')))* '-->'
+ [16]                 PI ::= '<?' PITarget (S (Char* - (Char* '?>' Char*)))?
+                             '?>'
+ [18]             CDSect ::= CDStart CData CDEnd
+ [19]            CDStart ::= '<![CDATA['
+ [20]              CData ::= (Char* - (Char* ']]>' Char*))
+ [21]              CDEnd ::= ']]>'
+ [22]             prolog ::= XMLDecl? Misc* (doctypedecl Misc*)?
+ [23]            XMLDecl ::= '<?xml' VersionInfo EncodingDecl? SDDecl? S? '?>'
+ [24]        VersionInfo ::= S 'version' Eq ("'" VersionNum "'" |
+                             '"' VersionNum '"')
+ [32]             SDDecl ::= S 'standalone' Eq (("'" ('yes' | 'no') "'")
+                             | ('"' ('yes' | 'no') '"'))
+ [40]               STag ::= '<' Name (S Attribute)* S? '>'
+ [41]          Attribute ::= Name Eq AttValue
+ [42]               ETag ::= '</' Name S? '>'
+ [44]       EmptyElemTag ::= '<' Name (S Attribute)* S? '/>'
+ [66]            CharRef ::= '&#' [0-9]+ ';' | '&#x' [0-9a-fA-F]+ ';'
+ [68]          EntityRef ::= '&' Name ';'
+ [80]       EncodingDecl ::= S 'encoding' Eq
+                             ('"' EncName '"' | "'" EncName "'")
+
+
+XMLScan::XMLParser $B$G<BAu$5$l$F$$$k$b$N!#(B
+
+  [1]           document ::= prolog element Misc*
+ [17]           PITarget ::= Name - (('X' | 'x') ('M' | 'm') ('L' | 'l'))
+ [43]            content ::= CharData? ((element | Reference | CDSect | PI
+                             | Comment) CharData?)*
+ [39]            element ::= EmptyElemTag | STag content ETag
+
+
+
+
+== $B<BAu$7$F$$$k$,!"B>$N@8@.5,B'Cf$K%$%s%i%$%s$GE83+$7$?$b$N!#(B
+
+XMLScan::XMLScanner $B$G<BAu$5$l$F$$$k$b$N!#(B
+
+ [11]      SystemLiteral ::= ('"' [^"]* '"') | ("'" [^']* "'")
+ [12]       PubidLiteral ::= '"' PubidChar* '"' | "'" (PubidChar - "'")* "'"
+ [25]                 Eq ::= S? '=' S?
+ [27]               Misc ::= Comment | PI | S
+ [67]          Reference ::= EntityRef | CharRef
+ [75]         ExternalID ::= 'SYSTEM' S SystemLiteral
+                             | 'PUBLIC' S PubidLiteral S SystemLiteral
+
+
+
+
+== $B<BAu$7$F$$$k$,!"@8@.5,B'$K<c43JQ99$r2C$($F$$$k$b$N!#(B
+
+XMLScan::XMLScanner $B$O(B ExternalID $B$N@8@.5,B'$r(B
+    ExternalID ::= 'SYSTEM' S SystemLiteral
+                   | 'PUBLIC' S PubidLiteral S SystemLiteral?
+$B$H$7$F2r@O!#(BXMLScan::XMLParser $B$,(B SystemLiteral $B$r;}$?$J$$8x3+<1JL;R$N(B
+$B%A%'%C%/$r9T$&!#(B
+
+ [28]        doctypedecl ::= '<!DOCTYPE' S Name (S ExternalID)? S?
+                             ('[' (markupdecl | DeclSep)* ']' S?)? '>'
+
+
+
+
+== $BL$Ce<j$@$,Ce<jM=Dj$N$b$N!#(B
+
+$BFbIt(B DTD $B%5%V%;%C%H$K4X$9$k$b$N!#(B
+
+  [9]        EntityValue ::= '"' ([^%&"] | PEReference | Reference)* '"'
+                             | ("'" ([^%&'] | PEReference | Reference)* "'")
+ [28a]           DeclSep ::= PEReference | S
+ [29]         markupdecl ::= elementdecl | AttlistDecl | EntityDecl
+                             | NotationDecl | PI | Comment
+ [45]        elementdecl ::= '<!ELEMENT' S Name S contentspec S? '>'
+ [52]        AttlistDecl ::= '<!ATTLIST' S Name AttDef* S? '>'
+ [53]             AttDef ::= S Name S AttType S DefaultDecl
+ [54]            AttType ::= StringType | TokenizedType | EnumeratedType
+ [55]         StringType ::= 'CDATA'
+ [56]      TokenizedType ::= 'ID'
+ [57]     EnumeratedType ::= NotationType | Enumeration
+ [58]       NotationType ::= 'NOTATION' S '(' S? Name (S? '|' S? Name)* S? ')'
+ [59]        Enumeration ::= '(' S? Nmtoken (S? '|' S? Nmtoken)* S? ')'
+ [60]        DefaultDecl ::= '#REQUIRED' | '#IMPLIED' | (('#FIXED' S)? AttValue
+ [69]        PEReference ::= '%' Name ';'
+ [70]         EntityDecl ::= GEDecl | PEDecl
+ [71]             GEDecl ::= '<!ENTITY' S Name S EntityDef S? '>'
+ [72]             PEDecl ::= '<!ENTITY' S '%' S Name S PEDef S? '>'
+ [73]          EntityDef ::= EntityValue | (ExternalID NDataDecl?)
+ [74]              PEDef ::= EntityValue | ExternalID
+ [76]          NDataDecl ::= S 'NDATA' S Name
+ [82]       NotationDecl ::= '<!NOTATION' S Name S (ExternalID | PublicID)
+                             S? '>'
+ [83]           PublicID ::= 'PUBLIC' S PubidLiteral
+
+
+
+== $BCe<jM=DjL5$+$C$?$1$ICe<j$7$A$c$C$?$b$N!#(B
+
+$BMWAGFbMF%b%G%k$N@53N$J9=J82r@O$O$d$i$J$$$+$b!#(B
+$B$C$F8@$C$F$?$N$K(B($B>P(B)$B!#(B> contentspec.ry
+
+ [46]        contentspec ::= 'EMPTY' | 'ANY' | Mixed | children
+ [47]           children ::= (choice | seq) ('?' | '*' | '+')?
+ [48]                 cp ::= (Name | choice | seq) ('?' | '*' | '+')?
+ [49]             choice ::= '(' S? cp ( S? '|' S? cp )+ S? ')'
+ [50]                seq ::= '(' S? cp ( S? ',' S? cp )* S? ')'
+ [51]              Mixed ::= '(' S? '#PCDATA' (S? '|' S? Name)* S? ')*'
+                             | '(' S? '#PCDATA' S? ')'
+
+
+
+== $BL$Ce<j$GCe<jM=Dj$N$J$$$b$N!#(B
+
+$B30It<BBN$K4X$9$k$b$NEy!#(B
+
+$B$3$l$i$O30It(BDTD$B%5%V%;%C%H$G$J$$$HMQL5$7!#(B
+
+ [30]          extSubset ::= TextDecl? extSubsetDecl
+ [31]      extSubsetDecl ::= ( markupdecl | conditionalSect | DeclSep)*
+ [61]    conditionalSect ::= includeSect | ignoreSect
+ [62]        includeSect ::= '<![' S? 'INCLUDE' S? '[' extSubsetDecl ']]>'
+ [63]         ignoreSect ::= '<![' S? 'IGNORE' S? '[' ignoreSectContents* ']]>'
+ [64] ignoreSectContents ::= Ignore ('<![' ignoreSectContents ']]>' Ignore)*
+ [65]             Ignore ::= Char* - (Char* ('<![' | ']]>') Char*)
+
+$B$3$l$i$O30It2r@OBP>]<BBN$G$J$$$HMQL5$7!#(B
+
+ [77]           TextDecl ::= '<?xml' VersionInfo? EncodingDecl S? '?>'
+ [78]       extParsedEnt ::= TextDecl? content
+
+$B$3$l$i$OB0@-CM$NBEEv@-$r8!>Z$7$J$$$J$iMQL5$7!#(B
+
+  [6]              Names ::= Name (S Name)*
+  [8]           Nmtokens ::= Nmtoken (S Nmtoken)*
+
+
+
+
+== $BJ8;z$dL>A0$K4X$9$k$b$N!#(B
+
+/[ \t\n\r]+/ $B$H$7$FB>$N@8@.5,B'Fb$KE83+$5$l$k!#(B
+Ruby $B$N(B \s $B$O(B #xC $B$r4^$s$G$7$^$&$N$G87L)$K$O(B S $B$H0lCW$7$J$$$3$H$KCm0U!#(B
+
+  [3]                  S ::= (#x20 | #x9 | #xD | #xA)+
+
+
+$B0J2<$O87L)$K<BAu$9$k$HIi2Y$,$+$+$C$F7y$J$N$G%G%U%)%k%H$G$OL58z!#(B
+$B%*%W%7%g%s$H$7$FA*Br$9$k$3$H$,$G$-$k!#(B
+
+
+XMLScan::XMLChar::valid_char?
+
+  [2]               Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
+                             | [#x10000-#x10FFFF]
+
+XMLScan::XMLChar::valid_nmtoken?
+
+  [4]           NameChar ::= Letter | Digit | '.' | '-' | '_' | ':'
+                             | CombiningChar | Extender
+  [7]            Nmtoken ::= (NameChar)+
+ [84]             Letter ::= BaseChar | Ideographic
+ [85]           BaseChar ::= ...
+ [86]        Ideographic ::= ...
+ [87]      CombiningChar ::= ...
+ [88]              Digit ::= ...
+ [89]           Extender ::= ...
+
+XMLScan::XMLChar::valid_name?
+
+  [5]               Name ::= (Letter | '_' | ':') (NameChar)*
+
+XMLScan::XMLChar::valid_pubid?
+
+ [13]          PubidChar ::= #x20 | #xD | #xA | [a-zA-Z0-9]
+                             | [-'()+,./:=?;!*#@$_%]
+
+XMLScan::XMLChar::valid_versionnum?
+
+ [26]         VersionNum ::= ([a-zA-Z0-9_.:] | '-')+
+
+XMLScan::XMLChar::valid_encname?
+
+ [81]            EncName ::= [A-Za-z] ([A-Za-z0-9._] | '-')*

data/redist/xmlscan/memo/contentspec.ry

@@ -0,0 +1,335 @@
+# $Id: contentspec.ry,v 1.2 2003/01/19 22:38:46 katsu Exp $
+
+#[46]        contentspec ::= 'EMPTY' | 'ANY' | Mixed | children
+#[47]           children ::= (choice | seq) ('?' | '*' | '+')?
+#[48]                 cp ::= (Name | choice | seq) ('?' | '*' | '+')?
+#[49]             choice ::= '(' S? cp ( S? '|' S? cp )+ S? ')'
+#[50]                seq ::= '(' S? cp ( S? ',' S? cp )* S? ')'
+#[51]              Mixed ::= '(' S? '#PCDATA' (S? '|' S? Name)* S? ')*'
+#                            | '(' S? '#PCDATA' S? ')'
+# �����Ǥ�դ�������ǽ�������� ? * + ��ľ���˶�����֤��ʤ���
+
+
+# + positive closure (������)
+# * kleene closure (����)
+# ?
+# | union (����)
+# , concatenation (Ϣ��)
+
+
+class ContentSpecParser
+
+options no_result_var
+
+rule
+
+  contentspec:	'EMPTY'
+		| 'ANY'
+		{ @result.push :ANY }
+		| mixed
+		| children
+
+  mixed:	'(' mixedelem ')*'
+		{ @result.push :* }
+		| '(' '#PCDATA' ')'
+		{ @result.push :PCDATA }
+		| '(' '#PCDATA' ')*'
+		{ @result.push :PCDATA, :- }
+
+  mixedelem:	'#PCDATA' '|' Name
+		{ @result.push :PCDATA, val[2], :| }
+		| mixedelem '|' Name
+		{ @result.push val[2], :| }
+
+  children:	choice
+		| seq
+
+  choice:	'(' choicelist listend
+
+  choicelist:	cp '|' cp
+		{ @result.push :| }
+		| choicelist '|' cp
+		{ @result.push :| }
+
+  seq:		'(' seqlist listend
+
+  seqlist:	cp
+		| seqlist ',' cp
+		{ @result.push :** }
+
+  listend:	')'
+		| ')?'
+		{ @result.push :- }
+		| ')*'
+		{ @result.push :* }
+		| ')+'
+		{ @result.push :+ }
+
+  cp:		Name
+		{ @result.push val[0] }
+		| NameQ
+		{ @result.push val[0], :- }
+		| NameA
+		{ @result.push val[0], :* }
+		| NameP
+		{ @result.push val[0], :+ }
+		| choice
+		| seq
+
+end
+
+
+---- inner ----
+
+  def next_token
+    @tokens.shift
+  end
+
+  def parse(src)
+    @tokens = []
+    first = true
+    src.scan(/([^ \t\n\r(|,)?*+]+[?*+]?)|([(|,]|\)[?*+]?)|([^ \t\n\r][\S\s]*)/
+             ) { |name,delim,error|
+      if name then
+        if first and (name == 'EMPTY' or name == 'ANY') then
+          @tokens.push [ name, name ]
+        elsif name == '#PCDATA' then
+          @tokens.push [ name, name ]
+        else
+          c = name[-1]
+          if c == ?? then
+            name.chop!
+            @tokens.push [ :NameQ, name ]
+          elsif c == ?* then
+            name.chop!
+            @tokens.push [ :NameA, name ]
+          elsif c == ?+ then
+            name.chop!
+            @tokens.push [ :NameP, name ]
+          else
+            @tokens.push [ :Name, name ]
+          end
+        end
+      elsif delim then
+        @tokens.push [ delim, delim ]
+      else
+        raise error
+      end
+    }
+    @tokens.push [ false, nil ]
+    @yydebug = true
+    @result = []
+    do_parse
+    @result
+  end
+
+---- footer ----
+
+
+
+
+module XMLScan
+
+  class DTDScanner
+
+    def parse_error(msg)
+      print "  #{caller[0]}: ", msg, "\n"
+    end
+
+    ClosureMark = { ?? => :-, ?* => :*, ?+ => :+ }
+    SeqDelim = { ?, => :**, ?| => :| }
+
+    def scan_contentspec(src)
+      dst = []
+      parenstack = []
+      pcdata = false
+      parenallow = true
+      if src == 'EMPTY' then
+        return []
+      elsif src == 'ANY' then
+        return [:ANY]
+      end
+      src.scan(
+          /([^ \t\n\r(|,)?*+%;]+[?*+]?)|([(|,]|\)[?*+]?)|([^ \t\n\r][\S\s]*)/
+               ) { |name,delim,error|
+        if name then
+          if not parenallow or parenstack.empty? then
+            parse_error "parse error at `#{name}'"
+            break
+            # RECOVERY-01: insert a `(' before Name. Also insert `,(' if needed
+            parenstack.push nil if parenstack.empty?
+            parenstack[-1] = (:**) unless parenstack.last
+            parenstack.push nil
+          end
+          mark = ClosureMark[name[-1]]
+          name.chop! if mark
+          if name == '#PCDATA' then
+            if mark or not dst.empty? or parenstack.size > 1 then
+              parse_error "parse error at `#{name}#{mark&&mark.chr}'"
+              break
+              dst.push '#PCDATA'    # RECOVERY-02: regard as an element type.
+              dst.push mark if mark
+              mark = parenstack.last
+              dst.push mark if mark
+            else
+              dst.push :PCDATA
+              pcdata = true
+              parenstack[-1] = :|
+            end
+          else
+            dst.push name
+            dst.push mark if mark
+            mark = parenstack.last
+            dst.push mark if mark
+          end
+          parenallow = false
+        elsif delim then
+          mark = delim[1]
+          delim = delim[0]
+          if delim == ?( then
+            if not parenallow or pcdata then
+              parse_error "parse error at `('"
+              break
+              if pcdata then         # RECOVERY-03: allow it with limitation.
+                parenstack.push :|
+              else                   # RECOVERY-04: insert a `,' before `('.
+                parenstack[-1] = (:**) unless parenstack.last
+                parenstack.push nil
+                parenallow = true
+              end
+            else
+              parenstack.push nil
+            end
+          elsif delim == ?) then
+            if parenallow then
+              parse_error "parse error at `)#{mark&&mark.chr}'"
+              parenstack.pop ; break
+              mark = nil         # RECOVERY-05: allow it but ignore any marks.
+              parenallow = false
+            end
+            if parenstack.empty? then
+              parse_error "unbalanced `)'"
+              break
+              # RECOVERY-06: allow it.
+            end
+            parenstack.pop
+            if pcdata then
+              if dst.size > 1 then
+                unless parenstack.empty? then
+                  # here is continuation of RECOVERY-03: ignore any marks.
+                else
+                  unless mark == ?* then
+                    parse_error "parse error at `)#{mark&&mark.chr}'"
+                    break
+                    # RECOVERY-07: replace the mark with a `*'.
+                  end
+                  dst.push :*
+                end
+              elsif mark then
+                if mark == ?* then
+                  dst.push :-
+                else
+                  parse_error "parse error at `)#{mark&&mark.chr}'"
+                  break
+                  # RECOVERY-08: ignore it.
+                end
+              end
+            else
+              mark = ClosureMark[mark]
+              dst.push mark if mark
+              mark = parenstack.last
+              dst.push mark if mark
+            end
+          else  # | or ,
+            if parenallow or parenstack.empty? then
+              parse_error "parse error at `#{delim.chr}'"
+              break
+              # RECOVERY-09: ignore it.
+            else
+              mark = SeqDelim[delim]
+              if parenstack.last then
+                unless parenstack.last == mark then
+                  parse_error "parse error at `#{delim.chr}'"
+                  break
+                  # RECOVERY-10: regard that it is same as the previous one.
+                end
+              else
+                parenstack[-1] = mark
+              end
+              parenallow = true
+            end
+          end
+        elsif error then
+          parse_error "parse error `#{error}'"
+          break
+          parenallow = false     # RECOVERY-11: ignore all after.
+          parenstack.clear
+        end
+      }
+      #parse_error "unexpected termination of content model" if parenallow
+      parse_error "unbalanced `('" unless parenstack.empty?
+      dst
+    end
+
+  end
+
+end
+
+
+
+Examples = [
+  "EMPTY",
+  "ANY",
+  "(a,b,c)",
+  "(a,b*,c)",
+  "(a,b*,c+)",
+  "(a,(b*,c)+)",
+  "(a,(b*|(c,e)?)+,d)+",
+  "(a,(b,(c,(d,(e)))))",
+  "(a,b,c,d,e)",
+  "(((((a),b),c),d),e)",
+  "(#PCDATA|a|b|c)",
+  "(#PCDATA|a)*",
+  "(#PCDATA|a|b|c)*",
+  "(a|#PCDATA|a|b|c)*",
+  "()*",
+  "(a,)?",
+  "((a,b,),c)?",
+  "(a))",
+  "((a))",
+  "(a)b)",
+  "(a(b))",
+  "(a)",
+  "(a)*",
+  "(#PCDATA)",
+  "(#PCDATA)*",
+  "(#PCDATA)?",
+  "(a,(#PCDATA|b|c)*,d)",
+  "a*",
+  "a|b",
+  "EMPTY|b",
+  '(a|b,c)',
+  '(#PCDATA|(a|b)|c)*',
+  '(#PCDATA|a|(b|c))*',
+  ',a',
+  '(a)(b,c)',
+  '(#PCDATA)(a,b)',
+  '(a,,b)',
+]
+
+Examples.each { |s|
+  p s
+  puts
+  begin
+    print '  ', ContentSpecParser.new.parse(s).inspect, "\n"
+  rescue => e
+    print "  ERROR: #{caller[0]}: ", e.message.strip, " (#{e.class.name})\n"
+  end
+  puts
+  begin
+    print '  ', XMLScan::DTDScanner.new.scan_contentspec(s).inspect, "\n"
+  rescue => e
+    print "  ERROR: #{caller[0]}: ", e.message, " (#{e.class.name})\n"
+  end
+  puts
+}