tdp4r 1.4.0 → 1.4.1

data/doc/guide.txt

@@ -146,5 +146,5 @@ Parser Combinators
 StringTokenizer
 -----------------
 There is a simple tokenizer called TDPUtils::StringTokenizer in the library
-"tdputils".
+"tdp/utils".
 (See MyParser#parse in sample2.rb)

data/lib/tdp.rb

@@ -88,7 +88,6 @@ module TDParser
 
     def recover(buff, ts)
       buff.each{|b| ts.unshift(b)}
-      buff.clear()
     end
   end
   include BufferUtils
@@ -169,7 +168,8 @@ module TDParser
       NegativeParser.new(self)
     end
 
-    def parse(tokens=nil, &blk)
+    def parse(tokens=nil, buff=nil, &blk)
+      buff ||= TokenBuffer.new()
       if( blk.nil? )
         if( tokens.respond_to?(:shift) && tokens.respond_to?(:unshift) )
           @tokens = tokens
@@ -181,7 +181,7 @@ module TDParser
       else
         @tokens = TokenGenerator.new(&blk)
       end
-      r = call(@tokens, TokenBuffer.new())
+      r = call(@tokens, buff)
       if( r.nil? )
         nil
       else
@@ -685,15 +685,15 @@ module TDParser
   end
 
   class BackrefParser < ReferenceParser
-    attr_reader :reference, :equality
+    attr_reader :label, :equality
 
-    def initialize(ref, eqsym)
-      @reference = ref
+    def initialize(label, eqsym)
+      @label = label
       @equality  = eqsym
     end
 
     def call(tokens, buff)
-      ys = buff.map[@reference]
+      ys = buff.map[@label]
       if (ys.nil? || ys.empty?)
         nil
       else
@@ -702,12 +702,12 @@ module TDParser
     end
 
     def to_s()
-      "<backref:#{@reference}>"
+      "<backref:#{@label}>"
     end
 
     def ==(r)
       super(r) &&
-      (@reference == r.reference) &&
+      (@label == r.label) &&
       (@equality == r.equality)
     end
   end
@@ -735,11 +735,40 @@ module TDParser
 
     def ==(r)
       super(r) &&
-      (@stack == r.stack) &&
+      (@stack.object_id == r.stack.object_id) &&
       (@equality == r.equality)
     end
   end
 
+  class ConditionParser < Parser
+    attr_reader :condition
+
+    def initialize(&condition)
+      @condition = condition
+    end
+
+    def call(tokens, buff)
+      if (res = @condition.call(buff.map))
+        Sequence[res]
+      else
+        nil
+      end
+    end
+
+    def to_s()
+      "<condition:#{@condition}>"
+    end
+
+    def ==(r)
+      super(r) &&
+      (@condition == r.condition)
+    end
+
+    def same?(r)
+      false
+    end
+  end
+
   class StateParser < Parser
     attr_reader :state
 
@@ -809,6 +838,11 @@ module TDParser
   end
   alias fail fail_rule
 
+  def condition_rule(&b)
+    ConditionParser.new(&b)
+  end
+  alias condition condition_rule
+
   def leftrec(*rules, &act)
     f = Proc.new{|x|
       x[1].inject(x[0]){|acc,y|

data/lib/tdp/xml.rb

@@ -0,0 +1,184 @@
+require 'tdp'
+require 'rexml/parsers/pullparser'
+require 'rexml/document'
+
+module TDPXML
+  module XMLParser
+    class XMLTokenGenerator < TDParser::TokenGenerator
+      def initialize(src)
+        @xparser = REXML::Parsers::BaseParser.new(src)
+        super(){|g|
+          while(@xparser.has_next?)
+            e = @xparser.pull()
+            g.yield(e)
+          end
+        }
+      end
+    end
+
+    class XArray < Array
+      def ===(ary)
+        if super(ary)
+          return true
+        end
+        if !ary.is_a?(Array)
+          return false
+        end
+        each_with_index{|v,idx|
+          case ary[idx]
+          when v
+          else
+            return false
+          end
+        }
+        true
+      end
+    end
+
+    class XHash < Hash
+      def ===(h)
+        if super(h)
+          return true
+        end
+        if !h.is_a?(Hash)
+          return false
+        end
+        each{|k,v|
+          case h[k]
+          when v
+          else
+            return false
+          end
+        }
+        true
+      end
+    end
+
+    def start_element(name=String)
+      token(XArray[:start_element, name, Hash])
+    end
+
+    def end_element(name=String)
+      token(XArray[:end_element, name])
+    end
+
+    def element(elem=String, &inner)
+      if inner
+        crule = inner.call()|empty()
+      else
+        crule = empty()
+      end
+      start_element(elem) - crule - end_element(elem) >> Proc.new{|x|
+        name = x[0][1]
+        attrs = x[0][2]
+        node = REXML::Element.new()
+        node.name = name
+        node.attributes.merge!(attrs)
+        [node,x[1]]
+      }
+    end
+
+    def text(match=String)
+      token(XArray[:text, match]) >> Proc.new{|x|
+        REXML::Text.new(x[0][1])
+      }
+    end
+
+    def pi()
+      token(XArray[:processing_instruction, String, String]) >> Proc.new{|x|
+        REXML::Instruction.new(x[0][1],x[0][2])
+      }
+    end
+
+    def cdata(match=String)
+      token(XArray[:cdata, match]) >> Proc.new{|x|
+        REXML::CData.new(x[0][1])
+      }
+    end
+
+    def comment(match=String)
+      token(XArray[:comment, match]) >> Proc.new{|x|
+        REXML::Comment.new(x[0][1])
+      }
+    end
+
+    def xmldecl()
+      token(XArray[:xmldecl]) >> Proc.new{|x|
+        REXML::XMLDecl.new(x[0][1],x[0][2], x[0][3])
+      }
+    end
+
+    def start_doctype(name=String)
+      token(XArray[:start_doctype, name])
+    end
+
+    def end_doctype()
+      token(XArray[:end_doctype])
+    end
+
+    def doctype(name=String, &inner)
+      if (inner)
+        crule = inner.call()|empty()
+      else
+        crule = empty()
+      end
+      start_doctype(name) - crule - end_doctype() >> Proc.new{|x|
+        node = REXML::DocType.new(x[0][1..-1])
+        [node, x[1]]
+      }
+    end
+
+    def externalentity(entity=String)
+      token(XArray[:externalentity, entity]) >> Proc.new{|x|
+        REXML::ExternalEntity.new(x[0][1])
+      }
+    end
+
+    def elementdecl(elem=String)
+      token(XArray[:elementdecl, elem]) >> Proc.new{|x|
+        REXML::ElementDecl.new(x[0][1])
+      }
+    end
+
+    def entitydecl(entity=String)
+      token(XArray[:entitydecl, elem]) >> Proc.new{|x|
+        REXML::Entity.new(x[0])
+      }
+    end
+
+    def attlistdecl(decl=String)
+      token(XArray[:attlistdecl]) >> Proc.new{|x|
+        REXML::AttlistDecl.new(x[0][1..-1])
+      }
+    end
+
+    def notationdecl(decl=String)
+      token(XArray[:notationdecl]) >> Proc.new{|x|
+        REXML::NotationDecl.new(*x[0][1..-1])
+      }
+    end
+
+    def any_node(&b)
+      (element(&b) | doctype(&b) | text() | pi() | cdata() |
+       comment() | xmldecl() | externalentity() | elementdecl() |
+       entitydecl() | attlistdecl() | notationdecl()) >> Proc.new{|x| x[2]}
+    end
+
+    def dom_constructor(&act)
+      Proc.new{|x|
+        node = x[0][0]
+        child = x[0][1]
+        if (child.is_a?(Array))
+          child.each{|c| node.add(c) }
+        else
+          node.add(child)
+        end
+        if (act)
+          act[node]
+        else
+          node
+        end
+      }
+    end
+  end
+end

data/samples/sample2.rb

@@ -2,7 +2,7 @@
 # parsing four arithmetic expressions with tdputils.
 
 require 'tdp'
-require 'tdputils'
+require 'tdp/utils'
 
 class MyParser
   include TDParser

data/samples/sample3.rb

@@ -2,7 +2,7 @@
 # parsing four arithmetic expressions with tdputils.
 
 require 'tdp'
-require 'tdputils'
+require 'tdp/utils'
 
 class MyParser
   include TDParser

data/samples/sample4.rb

@@ -2,7 +2,7 @@
 # caching constructed grammars
 
 require 'tdp'
-require 'tdputils'
+require 'tdp/utils'
 
 class MyParser
   include TDParser

data/samples/sample5.rb

@@ -2,7 +2,7 @@
 # writing grammars in the substitution style.
 
 require 'tdp'
-require 'tdputils'
+require 'tdp/utils'
 
 parser = TDParser.define{|g|
   g.plus = "+"

data/samples/sample_expr.rb

@@ -2,7 +2,7 @@
 # writing grammars using chainl().
 
 require 'tdp'
-require 'tdputils'
+require 'tdp/utils'
 
 parser = TDParser.define{|g|
   g.plus = "+"

data/samples/sample_optimize.rb

@@ -33,13 +33,14 @@ puts(parser.rule1.to_s)
 puts(parser.rule2.to_s)
 puts(parser.rule3.to_s)
 
+N = 10
 Benchmark.bm{|x|
   buff = ["1","2"]
   b = ["b"]
   for i in [5,10,15]
     puts("--")
-    x.report{ $r1 = parser.rule1.parse(buff*i + b*i) }
-    x.report{ $r2 = parser.rule2.parse(buff*i + b*i) }
-    x.report{ $r3 = parser.rule3.parse(buff*i + b*i) }
+    x.report{ N.times{ $r1 = parser.rule1.parse(buff*i + b*i) } }
+    x.report{ N.times{ $r2 = parser.rule2.parse(buff*i + b*i) } }
+    x.report{ N.times{ $r3 = parser.rule3.parse(buff*i + b*i) } }
   end
 }

data/samples/sample_xml.rb

@@ -1,123 +1,52 @@
 require 'tdp'
-require 'tdputils'
-require 'rexml/parsers/pullparser'
-require 'rexml/document'
-
-class Array
-  def ===(ary)
-    if super(ary)
-      return true
-    end
-    if !ary.is_a?(Array)
-      return false
-    end
-    each_with_index{|v,idx|
-      case ary[idx]
-      when v
-      else
-        return false
-      end
-    }
-    true
-  end
-end
-
-class Hash
-  def ===(h)
-    if super(h)
-      return true
-    end
-    if !h.is_a?(Hash)
-      return false
-    end
-    each{|k,v|
-      case h[k]
-      when v
-      else
-        return false
-      end
-    }
-    true
-  end
-end
-
-module XMLParser
-  def xml_stag(name)
-    token([:start_element, name, Hash])
-  end
-  alias stag xml_stag
-
-  def xml_etag(name)
-    token([:end_element, name])
-  end
-  alias etag xml_etag
-
-  def dom_element(elem, &inner)
-    stag(elem) - (inner.call()|empty()) - etag(elem)
-  end
-  alias element dom_element
-
-  def dom_filter(&act)
-    Proc.new{|x|
-      name = x[0][1]
-      attrs = x[0][2]
-      node = REXML::Element.new()
-      node.name = name
-      node.attributes.merge!(attrs)
-      act[node,x[1]]
-    }
-  end
-  alias filter dom_filter
-
-  def dom_construct(&act)
-    dom_filter{|node,child|
-      if (child.is_a?(Array))
-        child.each{|c| node.add(c) }
-      else
-        node.add(child)
-      end
-      if (act)
-        act[node]
-      else
-        node
-      end
-    }
-  end
-  alias construct dom_construct
-end
+require 'tdp/utils'
+require 'tdp/xml'
 
 translator = TDParser.define{|g|
-  extend XMLParser
+  extend TDPXML::XMLParser
 
   g.xml =
     element("a"){
       element("b"){
-        g.xml*0 >> Proc.new{|x| x[0].collect{|y| y[0]} }
-      } >> construct{|node| node.name = "bar"; node }
-    } >> construct{|node| node.name = "foo"; node } |
+        g.xmlseq
+      } >> dom_constructor{|node| node.children() }
+    } >> dom_constructor{|node| node.name = "AB"; node } |
     element(String){
-      g.xml*0 >> Proc.new{|x| x[0].collect{|y| y[0]} }
-    } >> construct{|node|
-      node.name = node.name.upcase()
-      node
-    } |
-    ~etag(String) - any() - g.xml >> Proc.new{|x| x[2]}
+      g.xmlseq
+    } >> dom_constructor{|node|
+           node.name = node.name.upcase()
+           node
+         } |
+    doctype{
+      g.xmlseq
+    } >> dom_constructor{|node| node} |
+    text >> Proc.new{|x| x[0]} |
+    elementdecl >> Proc.new{|x| x[0]} |
+    xmldecl >> Proc.new{|x| x[0]} |
+    comment >> Proc.new{|x| x[0]} |
+    any_node() >> Proc.new{|x| x[0] }
+
+  g.xmlseq =
+    g.xml()*0 >> Proc.new{|x| x[0].collect{|y| y[0]}} |
 
   def translate(src)
-    xparser = REXML::Parsers::BaseParser.new(src)
-    xml.parse{|g|
-      while(xparser.has_next?)
-        g.yield(xparser.pull())
-      end
-    }
+    gen = TDPXML::XMLParser::XMLTokenGenerator.new(src)
+    xmlseq.parse(gen)
   end
 }
 
-puts(translator.translate(<<EOS))
+seq = translator.translate(<<EOS)
 <?xml version="1.0" ?>
+<!DOCTYPE body [
+ <!ELEMENT body (#PCDATA, strong*)>
+ <!ELEMENT strong (#PCDATA)>
+ ]>
 <list>
+  <!-- comment -->
   <a><b><c>hoge</c></b></a>
   <b>b?</b>
 </list>
 EOS
-# => "<LIST><foo><bar><C></C></bar></foo><B></B></LIST>"
+doc = REXML::Document.new()
+seq.each{|x| doc.add(x) }
+puts(doc)

data/test/test_tdp.rb

@@ -1,6 +1,7 @@
 require 'test/unit'
 require 'tdp'
-require 'tdputils'
+require 'tdp/utils'
+require 'tdp/xml'
 
 class Tokens
   include Enumerable
@@ -367,6 +368,14 @@ class TestTDParser < Test::Unit::TestCase
     assert_equal(["a","b",[["a","b"],["a","b"]]], rule.parse(buff))
   end
 
+  def test_backref4()
+    rule = (token(/\w/) - token(/\w/))/:x - (token("-")|backref(:x))*0 >> proc{|x| x}
+    assert_equal(["a","b",[["a","b"],["a","b"]]],
+                 rule.parse(["a","b","a","b","a","b"]))
+    assert_equal(["a","b",[["-"],["a","b"]]],
+                 rule.parse(["a","b","-","a","b"]))
+  end
+
   def test_stackref1()
     buff = ["a","b","a"]
     stack = []
@@ -456,6 +465,25 @@ class TestTDParser < Test::Unit::TestCase
     assert_equal(0, rule.parse(buff))
   end
 
+  def test_condition1()
+    rule = condition{|m|m["n"]=20} - condition{|m|m["n"]} >> Proc.new{|x| x}
+    assert_equal([20,20], rule.parse([]))
+  end
+
+  def test_condition2()
+    rule = condition{|m|m["n"]=20} - condition{|m|m["n"]>20} >> Proc.new{|x| x}
+    assert_equal(nil, rule.parse([]))
+  end
+
+  def test_condition3()
+    rule =
+      condition{|m|m["n"]=20} -
+        (token("a") - condition{|m|m["n"]>20} |
+         token("b") - condition{|m|m["n"]>10}) >> Proc.new{|x| x}
+    assert_equal(nil, rule.parse(["a"]))
+    assert_equal([20,"b",true], rule.parse(["b"]))
+  end
+
   def test_rule1()
     expr = "1 + 2"
     assert_equal(3, @calc.parse(expr))

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: tdp4r
 version: !ruby/object:Gem::Version 
-  version: 1.4.0
-date: 2006-07-22 00:00:00 +09:00
+  version: 1.4.1
+date: 2006-07-23 00:00:00 +09:00
 summary: TDP4R is a top-down parser library that consists of parser combinators and utility functions.
 require_paths: 
 - lib
@@ -29,8 +29,10 @@ post_install_message:
 authors: 
 - Takaaki Tateishi
 files: 
+- lib/tdp
 - lib/tdp.rb
-- lib/tdputils.rb
+- lib/tdp/utils.rb
+- lib/tdp/xml.rb
 - samples/sample1.rb
 - samples/sample2.rb
 - samples/sample3.rb