tdp4r 1.3.3 → 1.4.0

data/lib/tdp.rb

@@ -10,8 +10,8 @@ module TDParser
   end
 
   class TokenGenerator < Generator
-    def initialize(*args)
-      super(*args)
+    def initialize(*args, &block)
+      super(*args, &block)
       @buffer = []
     end
 
@@ -93,141 +93,80 @@ module TDParser
   end
   include BufferUtils
 
-  class Rule < Proc
+  class Parser
     include BufferUtils
+    include TDParser
+
+    def to_proc()
+      Proc.new{|*x| self.call(*x) }
+    end
+
+    def to_s()
+      "??"
+    end
+
+    def call(*args)
+    end
+
+    #def [](*args)
+    #  call(*args)
+    #end
+
+    def optimize(default=false)
+      self.dup()
+    end
+
+    def ==(r)
+      false
+    end
+
+    def same?(r)
+      self == r
+    end
 
     def -(r)
-      Rule.new{|ts, buff|
-        if( (x = self[ts, buff]).nil? )
-          nil
-        else
-          if( (y = r[ts, buff]).nil? )
-            nil
-          else
-            x + y
-          end
-        end
-      }
+      ConcatParser.new(self,r)
+    end
+
+    def +(r)
+      ParallelParser.new(self,r)
     end
 
     def |(r)
-      Rule.new{|ts, buff|
-        b = prepare(buff)
-        if( (x = self[ts, b]).nil? )
-          recover(b, ts)
-          r[ts, buff]
-        else
-          buff.insert(0, *b)
-          x
-        end
-      }
+      ChoiceParser.new(self,r).optimize(true)
     end
 
-    def *(n)
-      if( n.is_a?(Range) )
-        range = n
+    def *(range)
+      if( range.is_a?(Range) )
         n = range.min
       else
+        n = range
         range = nil
       end
-      Rule.new{|ts, buff|
-        x  = true
-        xs = []
-        while( n > 0 )
-          n -= 1
-          b = prepare(buff)
-          if( (x = self[ts, b]).nil? )
-            recover(b, ts)
-            break
-          else
-            buff.insert(0, *b)
-            xs.push(x)
-          end
-        end
-        if ( x.nil? )
-          nil
-        else
-          if( range )
-            range.each{
-              while( true )
-                y = x
-                b = prepare(buff)
-                if( (x = self[ts, b]).nil? )
-                  recover(b, ts)
-                  x = y
-                  break
-                else
-                  buff.insert(0, *b)
-                  xs.push(x)
-                end
-              end
-            }
-          else
-            while( true )
-              y = x
-              b = prepare(buff)
-              if( (x = self[ts, b]).nil? )
-                recover(b, ts)
-                x = y
-                break
-              else
-                buff.insert(0, *b)
-                xs.push(x)
-              end
-            end
-          end
-          Sequence[xs]
-        end
-      }
+      IterationParser.new(self,n,range)
     end
 
     def >>(act)
-      Rule.new{|tokens, buff|
-        if( (x = self[tokens, buff]).nil? )
-          nil
-        else
-          x = TokenBuffer[*x]
-          x.map = buff.map
-          Sequence[act[x]]
-        end
-      }
+      ActionParser.new(self,act)
     end
 
-    def /(symbol)
-      Rule.new{|tokens, buff|
-        x = self[tokens, buff]
-        buff.map[symbol] = x
-        x
-      }
+    def /(label)
+      LabelParser.new(self,label)
     end
 
     def %(stack)
-      Rule.new{|tokens, buff|
-        x = self[tokens, buff]
-        stack.push(x)
-        x
-      }
+      StackParser.new(self,stack)
     end
 
     def >(symbol)
-      Rule.new{|tokens, buff|
+      Parser.new{|tokens, buff|
         buff[symbol] = buff.dup()
         self[tokens, buff]
       }
     end
 
     def ~@()
-      Rule.new{|tokens, buff|
-        b = prepare(buff)
-        r = self[tokens,b]
-        rev = b.reverse
-        recover(b, tokens)
-        if( r.nil? )
-          Sequence[Sequence[*rev]]
-        else
-          nil
-        end
-      }
+      NegativeParser.new(self)
     end
 
     def parse(tokens=nil, &blk)
@@ -242,7 +181,7 @@ module TDParser
       else
         @tokens = TokenGenerator.new(&blk)
       end
-      r = self[@tokens, TokenBuffer.new()]
+      r = call(@tokens, TokenBuffer.new())
       if( r.nil? )
         nil
       else
@@ -262,112 +201,611 @@ module TDParser
         self >> block
     end
   end
-  # end of Rule
+  # end of Parser
+
+  class NonTerminalParser < Parser
+    attr_reader :context, :symbol, :options
+    def initialize(context, sym, *options)
+      @context = context
+      @symbol = sym
+      @options = options
+    end
 
-  def rule(sym, *opts)
-    Rule.new{|tokens, buff|
+    def call(tokens, buff)
       res = nil
-      case sym
+      case @symbol
       when Symbol, String
-        res = __send__(sym,*opts)[tokens, buff]
-      when Rule
-        res = sym[tokens, buff]
-      end
-      if( block_given? && !res.nil? )
-        res = yield(res)
+        res = @context.__send__(@symbol,*@options).call(tokens, buff)
+      when Parser
+        res = @symbol.call(tokens, buff)
       end
       res
-    }
+    end
+
+    def ==(r)
+      (self.class == r.class) &&
+      (@context == r.context) &&
+      (@symbol == r.symbol) &&
+      (@options == r.options)
+    end
+
+    def to_s()
+      "#{@symbol.to_s}"
+    end
   end
 
-  def token(x, eqsym=:===)
-    Rule.new{|tokens, buff|
+  class TerminalParser < Parser
+    attr_reader :symbol, :equality
+
+    def initialize(obj, eqsym)
+      @symbol   = obj
+      @equality = eqsym
+    end
+
+    def call(tokens, buff)
       t = tokens.shift
       buff.unshift(t)
-      if( x.__send__(eqsym,t) || t.__send__(eqsym,x) )
-        t = yield(t) if( block_given? )
+      if( @symbol.__send__(@equality,t) || t.__send__(@equality,@symbol) )
         Sequence[t]
       else
         nil
       end
-    }
+    end
+
+    def ==(r)
+      (self.class == r.class) &&
+      (@symbol == r.symbol) &&
+      (@equality == r.equality)
+    end
+
+    def to_s()
+      "#{@symbol}"
+    end
   end
 
-  def __backref__(xs, eqsym)
-    x = xs.shift()
-    xs.inject(token(x, eqsym)){|acc,x|
-      case x
-      when Sequence
-        acc - __backref__(x, eqsym)
+  class CompositeParser < Parser
+    attr_accessor :parsers
+
+    def initialize(*parsers, &b)
+      @parsers = parsers
+    end
+
+    def optimize(default=false)
+      parser = dup()
+      parser.parsers = @parsers.collect{|x| x.optimize(default)}
+      parser
+    end
+
+    def ==(r)
+      (self.class == r.class) &&
+      (@parsers == r.parsers)
+    end
+
+    def same?(r)
+      super(r) &&
+      @parsers.zip(r.parsers).all?{|x,y| x.same?(y)}
+    end
+
+    def to_s()
+      "<composite: #{@parsers.collect{|x| x.to_s()}}>"
+    end
+  end
+
+  class ActionParser < CompositeParser
+    attr_reader :action
+
+    def initialize(parser, act)
+      @action = act
+      super(parser)
+    end
+
+    def call(tokens, buff)
+      if( (x = @parsers[0].call(tokens, buff)).nil? )
+        nil
       else
-        acc - token(x, eqsym)
+        x = TokenBuffer[*x]
+        x.map = buff.map
+        Sequence[@action[x]]
       end
-    }
+    end
+
+    def ==(r)
+      super(r) &&
+      (@action == r.action)
+    end
+
+    def to_s()
+      "(#{@parsers[0].to_s()} <action>)"
+    end
   end
 
-  def backref(x, eqsym=:===)
-    Rule.new{|tokens, buff|
-      ys = buff.map[x]
-      if (ys.nil? || ys.empty?)
+  class LabelParser < CompositeParser
+    attr_reader :label
+
+    def initialize(parser, label)
+      @label = label
+      super(parser)
+    end
+
+    def call(tokens, buff)
+      x = @parsers[0].call(tokens, buff)
+      buff.map[@label] = x
+      x
+    end
+
+    def ==(r)
+      super(r) &&
+      (@label == r.label)
+    end
+
+    def to_s()
+      "(#{@parsers[0].to_s()}/#{@label})"
+    end
+  end
+
+  class StackParser < CompositeParser
+    attr_reader :stack
+    def initialize(parser, stack)
+      @stack = stack
+      super(parser)
+    end
+
+    def call(tokens, buff)
+      x = @parsers[0].call(tokens, buff)
+      @stack.push(x)
+      x
+    end
+
+    def ==(r)
+      super(r) &&
+      (@stack == r.stack)
+    end
+
+    def same?(r)
+      false
+    end
+
+    def to_s()
+      "<stack:#{@stack.object_id}>"
+    end
+  end
+
+  class ConcatParser < CompositeParser
+    def initialize(r1, r2)
+      super(r1, r2)
+    end
+
+    def call(tokens, buff)
+      if( (x = @parsers[0].call(tokens, buff)).nil? )
         nil
       else
-        __backref__(ys.dup(), eqsym)[tokens,buff]
+        if( (y = @parsers[1].call(tokens, buff)).nil? )
+          nil
+        else
+          x + y
+        end
       end
-    }
+    end
+
+    def -(r)
+      @parsers[0] - (@parsers[1] - r)
+    end
+
+    def to_s()
+      "(#{@parsers[0].to_s()} #{@parsers[1].to_s()})"
+    end
   end
 
-  def stackref(stack, eqsym=:===)
-    Rule.new{|tokens, buff|
-      ys = stack.pop()
-      if (ys.nil? || ys.empty?)
+  class ChoiceParser < CompositeParser
+    def initialize(r1, r2)
+      super(r1, r2)
+    end
+
+    def call(tokens, buff)
+      b = prepare(buff)
+      if( (x = @parsers[0].call(tokens, b)).nil? )
+        recover(b, tokens)
+        @parsers[1].call(tokens, buff)
+      else
+        buff.insert(0, *b)
+        x
+      end
+    end
+
+    def to_s()
+      "(#{@parsers[0].to_s()} | #{@parsers[1].to_s()})"
+    end
+
+    def shared_sequence(r1, r2)
+      if (r1.is_a?(ConcatParser) && r2.is_a?(ConcatParser))
+        r11 = r1.parsers[0]
+        r12 = r1.parsers[1]
+        r21 = r2.parsers[0]
+        r22 = r2.parsers[1]
+        if (r11.same?(r21))
+          share,r12,r22, = shared_sequence(r12, r22)
+          if (share)
+            return [r11 - share, r12, r22]
+          else
+            return [r11, r12, r22]
+          end
+        end
+      end
+      [nil, r1, r2]
+    end
+
+    def optimize(default=false)
+      r1 = @parsers[0]
+      r2 = @parsers[1]
+      if (r1.is_a?(ActionParser))
+        act1 = r1.action
+        r1 = r1.parsers[0]
+      end
+      if (r2.is_a?(ActionParser))
+        act2 = r2.action
+        r2 = r2.parsers[0]
+      end
+      share,r12,r22, = shared_sequence(r1, r2)
+      if (share)
+        r = share - (r12 + r22)
+        if (act1)
+          if (act2)
+            r = r >> Proc.new{|x|
+              y0,y1,*ys = x.pop()
+              if (y0)
+                act1.call(x.push(*y0))
+              else
+                act2.call(x.push(*y1))
+              end
+            }
+          else
+            r = r >> Proc.new{|x|
+              y0,y1,*ys = x.pop()
+              if (y0)
+                act1.call(x.push(*y0))
+              end
+            }
+          end
+        else
+          if (act2)
+            r = r >> Proc.new{|x|
+              y0,y1,*ys = x.pop()
+              if (y1)
+                act2.call(x.push(*y1))
+              end
+            }
+          end
+        end
+        return r
+      end
+      if (default)
+        self.dup()
+      else
+        super(default)
+      end
+    end
+  end
+
+  class ParallelParser < CompositeParser
+    def initialize(r1, r2)
+      super(r1, r2)
+    end
+
+    def call(tokens, buff)
+      b = prepare(buff)
+      if( (x = @parsers[0].call(tokens, b)).nil? )
+        recover(b, tokens)
+        Sequence[Sequence[nil, @parsers[1].call(tokens, buff)]]
+      else
+        buff.insert(0, *b)
+        Sequence[Sequence[x, nil]]
+      end
+    end
+
+    def to_s()
+      "(#{@parsers[0].to_s()} + #{@parsers[1].to_s()})"
+    end
+  end
+
+  class IterationParser < CompositeParser
+    attr_reader :min, :range
+    def initialize(parser, n, range)
+      @min = n
+      @range = range
+      super(parser)
+    end
+
+    def call(ts, buff)
+      r = @parsers[0]
+      n = @min
+      x  = true
+      xs = []
+      while( n > 0 )
+        n -= 1
+        b = prepare(buff)
+        if( (x = r.call(ts, b)).nil? )
+          recover(b, ts)
+          break
+        else
+          buff.insert(0, *b)
+          xs.push(x)
+        end
+      end
+      if ( x.nil? )
         nil
       else
-        __backref__(ys.dup(), eqsym)[tokens,buff]
+        if( range )
+          range.each{
+            while( true )
+              y = x
+              b = prepare(buff)
+              if( (x = r.call(ts, b)).nil? )
+                recover(b, ts)
+                x = y
+                break
+              else
+                buff.insert(0, *b)
+                xs.push(x)
+              end
+            end
+          }
+        else
+          while( true )
+            y = x
+            b = prepare(buff)
+            if( (x = r.call(ts, b)).nil? )
+              recover(b, ts)
+              x = y
+              break
+            else
+              buff.insert(0, *b)
+              xs.push(x)
+            end
+          end
+        end
+        Sequence[xs]
       end
-    }
+    end
+
+    def to_s()
+      "(#{@parsers[0].to_s()})*#{@range ? @range.to_s : @min.to_s}"
+    end
+
+    def ==(r)
+      super(r) &&
+      (@min == r.min) &&
+      (@range == r.range)
+    end
   end
 
-  def state(s)
-    Rule.new{|tokens, buff|
-      if (buff.map[:state] == s)
-        Sequence[s]
+  class NegativeParser < CompositeParser
+    def initialize(parser)
+      super(parser)
+    end
+
+    def call(tokens, buff)
+      b = prepare(buff)
+      r = @parsers[0].call(tokens,b)
+      rev = b.reverse
+      recover(b, tokens)
+      if( r.nil? )
+        Sequence[Sequence[*rev]]
       else
         nil
       end
-    }
+    end
+
+    def to_s()
+      "~#{@parsers[0]}"
+    end
   end
 
-  def empty_rule()
-    Rule.new{|tokens, buff| Sequence[nil] }
+  class FailParser < Parser
+    def call(tokens, buff)
+      nil
+    end
+
+    def to_s()
+      "<fail>"
+    end
+
+    def ==()
+      (self.class == r.class)
+    end
   end
-  alias empty empty_rule
 
-  def any_rule()
-    Rule.new{|tokens, buff|
+  class EmptyParser < Parser
+    def call(tokens, buff)
+      Sequence[nil]
+    end
+
+    def to_s()
+      "<empty>"
+    end
+
+    def ==(r)
+      true
+    end
+  end
+
+  class AnyParser < Parser
+    def call(tokens, buff)
       t = tokens.shift
       if (t.nil?)
         nil
       else
         Sequence[t]
       end
-    }
+    end
+
+    def to_s()
+      "<any>"
+    end
+
+    def ==(r)
+      true
+    end
   end
-  alias any any_rule
 
-  def none_rule()
-    Rule.new{|tokens, buff|
+  class NoneParser < Parser
+    def call(tokens, buff)
       t = tokens.shift
       if (t.nil?)
         Sequence[nil]
       else
         nil
       end
-    }
+    end
+
+    def to_s()
+      "<none>"
+    end
+
+    def ==(r)
+      true
+    end
+  end
+
+  class ReferenceParser < Parser
+    def __backref__(xs, eqsym)
+      x = xs.shift()
+      xs.inject(token(x, eqsym)){|acc,x|
+        case x
+        when Sequence
+          acc - __backref__(x, eqsym)
+        else
+          acc - token(x, eqsym)
+        end
+      }
+    end
+
+    def same?(r)
+      false
+    end
+  end
+
+  class BackrefParser < ReferenceParser
+    attr_reader :reference, :equality
+
+    def initialize(ref, eqsym)
+      @reference = ref
+      @equality  = eqsym
+    end
+
+    def call(tokens, buff)
+      ys = buff.map[@reference]
+      if (ys.nil? || ys.empty?)
+        nil
+      else
+        __backref__(ys.dup(), @equality).call(tokens,buff)
+      end
+    end
+
+    def to_s()
+      "<backref:#{@reference}>"
+    end
+
+    def ==(r)
+      super(r) &&
+      (@reference == r.reference) &&
+      (@equality == r.equality)
+    end
+  end
+
+  class StackrefParser < ReferenceParser
+    attr_reader :stack, :equality
+
+    def initialize(stack, eqsym)
+      @stack = stack
+      @equality = eqsym
+    end
+
+    def call(tokens, buff)
+      ys = @stack.pop()
+      if (ys.nil? || ys.empty?)
+        nil
+      else
+        __backref__(ys.dup(), @equality).call(tokens,buff)
+      end
+    end
+
+    def to_s()
+      "<stackref:#{@stack.object_id}>"
+    end
+
+    def ==(r)
+      super(r) &&
+      (@stack == r.stack) &&
+      (@equality == r.equality)
+    end
+  end
+
+  class StateParser < Parser
+    attr_reader :state
+
+    def initialize(s)
+      @state = s
+    end
+
+    def call(tokens, buff)
+      if (buff.map[:state] == @state)
+        Sequence[@state]
+      else
+        nil
+      end
+    end
+
+    def to_s()
+      "<state:#{@state}>"
+    end
+
+    def ==(r)
+      super(r) &&
+      (@state == r.state)
+    end
+
+    def same?(r)
+      false
+    end
+  end
+
+  def rule(sym, *opts)
+    NonTerminalParser.new(self, sym, *opts)
+  end
+
+  def token(x, eqsym=:===)
+    TerminalParser.new(x, eqsym)
+  end
+
+  def backref(x, eqsym=:===)
+    BackrefParser.new(x, eqsym)
+  end
+
+  def stackref(stack, eqsym=:===)
+    StackrefParser.new(stack, eqsym)
+  end
+
+  def state(s)
+    StateParser.new(s)
+  end
+
+  def empty_rule(&b)
+    EmptyParser.new(&b)
+  end
+  alias empty empty_rule
+
+  def any_rule()
+    AnyParser.new()
+  end
+  alias any any_rule
+
+  def none_rule()
+    NoneParser.new()
   end
   alias none none_rule
 
   def fail_rule()
-    Rule.new{|tokens, buff| nil }
+    FailParser.new()
   end
   alias fail fail_rule
 
@@ -424,7 +862,7 @@ module TDParser
       sym = sym.to_s()
       if (sym[-1,1] == "=")
         case arg0
-        when Rule
+        when Parser
           self.class.instance_eval{
             define_method(sym[0..-2]){ arg0 }
           }

data/samples/sample_optimize.rb

@@ -0,0 +1,45 @@
+require 'tdp'
+require 'benchmark'
+require 'pp'
+
+# disable auto optimization
+module TDParser
+  class Parser
+    def |(r)
+      ChoiceParser.new(self, r)
+    end
+  end
+end
+
+parser = TDParser.define{|g|
+  f = Proc.new{|x| x.flatten}
+  g.rule1 =
+    token("1") - token("2") - rule1 - token("a") >> f |
+    token("1") - token("2") - rule1 - token("b") >> f |
+    empty()
+
+
+  g.rule2 =
+    (token("1") - token("2") - rule2 - token("a") >> f |
+     token("1") - token("2") - rule2 - token("b") >> f |
+     empty()).optimize()
+
+  g.rule3 =
+    (token("1") - token("2") - rule3 - (token("a")|token("b")) >> f |
+     empty())
+}
+
+puts(parser.rule1.to_s)
+puts(parser.rule2.to_s)
+puts(parser.rule3.to_s)
+
+Benchmark.bm{|x|
+  buff = ["1","2"]
+  b = ["b"]
+  for i in [5,10,15]
+    puts("--")
+    x.report{ $r1 = parser.rule1.parse(buff*i + b*i) }
+    x.report{ $r2 = parser.rule2.parse(buff*i + b*i) }
+    x.report{ $r3 = parser.rule3.parse(buff*i + b*i) }
+  end
+}

data/samples/sample_xml.rb

@@ -0,0 +1,123 @@
+require 'tdp'
+require 'tdputils'
+require 'rexml/parsers/pullparser'
+require 'rexml/document'
+
+class Array
+  def ===(ary)
+    if super(ary)
+      return true
+    end
+    if !ary.is_a?(Array)
+      return false
+    end
+    each_with_index{|v,idx|
+      case ary[idx]
+      when v
+      else
+        return false
+      end
+    }
+    true
+  end
+end
+
+class Hash
+  def ===(h)
+    if super(h)
+      return true
+    end
+    if !h.is_a?(Hash)
+      return false
+    end
+    each{|k,v|
+      case h[k]
+      when v
+      else
+        return false
+      end
+    }
+    true
+  end
+end
+
+module XMLParser
+  def xml_stag(name)
+    token([:start_element, name, Hash])
+  end
+  alias stag xml_stag
+
+  def xml_etag(name)
+    token([:end_element, name])
+  end
+  alias etag xml_etag
+
+  def dom_element(elem, &inner)
+    stag(elem) - (inner.call()|empty()) - etag(elem)
+  end
+  alias element dom_element
+
+  def dom_filter(&act)
+    Proc.new{|x|
+      name = x[0][1]
+      attrs = x[0][2]
+      node = REXML::Element.new()
+      node.name = name
+      node.attributes.merge!(attrs)
+      act[node,x[1]]
+    }
+  end
+  alias filter dom_filter
+
+  def dom_construct(&act)
+    dom_filter{|node,child|
+      if (child.is_a?(Array))
+        child.each{|c| node.add(c) }
+      else
+        node.add(child)
+      end
+      if (act)
+        act[node]
+      else
+        node
+      end
+    }
+  end
+  alias construct dom_construct
+end
+
+translator = TDParser.define{|g|
+  extend XMLParser
+
+  g.xml =
+    element("a"){
+      element("b"){
+        g.xml*0 >> Proc.new{|x| x[0].collect{|y| y[0]} }
+      } >> construct{|node| node.name = "bar"; node }
+    } >> construct{|node| node.name = "foo"; node } |
+    element(String){
+      g.xml*0 >> Proc.new{|x| x[0].collect{|y| y[0]} }
+    } >> construct{|node|
+      node.name = node.name.upcase()
+      node
+    } |
+    ~etag(String) - any() - g.xml >> Proc.new{|x| x[2]}
+
+  def translate(src)
+    xparser = REXML::Parsers::BaseParser.new(src)
+    xml.parse{|g|
+      while(xparser.has_next?)
+        g.yield(xparser.pull())
+      end
+    }
+  end
+}
+
+puts(translator.translate(<<EOS))
+<?xml version="1.0" ?>
+<list>
+  <a><b><c>hoge</c></b></a>
+  <b>b?</b>
+</list>
+EOS
+# => "<LIST><foo><bar><C></C></bar></foo><B></B></LIST>"

data/test/test_tdp.rb

@@ -325,7 +325,7 @@ class TestTDParser < Test::Unit::TestCase
   end
 
   def test_regex_match()
-    rule = token(/\d+/, :=~){|x| /(\d+)/ =~ x; $1.to_i } >> proc{|x| x[0] }
+    rule = token(/\d+/, :=~) >> proc{|x| x[0].to_i }
     assert_equal(10, rule.parse(["10"]))
   end
 
@@ -374,14 +374,14 @@ class TestTDParser < Test::Unit::TestCase
     assert_equal(["a","b","a"], rule.parse(buff))
   end
 
-  def test_backref2()
+  def test_stackref2()
     buff = ["a","b","c"]
     stack = []
     rule = token(/\w/)%stack - token("b") - stackref(stack) >> proc{|x| x}
     assert_equal(nil, rule.parse(buff))
   end
 
-  def test_backref3()
+  def test_stackref3()
     buff = ["a","b","a","b","a","b"]
     stack = []
     rule = (token(/\w/) - token(/\w/))%stack - (stackref(stack)%stack)*0 >> proc{|x| x}
@@ -393,6 +393,27 @@ class TestTDParser < Test::Unit::TestCase
     assert_equal(["a","b",[["a","b"]]], rule.parse(buff))
   end
 
+  def test_parallel1()
+    rule = token("a") - (token("b") + token("c")) >> Proc.new{|x| x}
+    assert_equal(["a",[["b"],nil]], rule.parse(["a","b"]))
+    assert_equal(["a",[nil,["c"]]], rule.parse(["a","c"]))
+  end
+
+  def test_parallel2()
+    rule = token("a") - token("b") - (token("c") + token("d")) >> Proc.new{|x| x}
+    assert_equal(["a","b",[["c"],nil]], rule.parse(["a","b","c"]))
+    assert_equal(["a","b",[nil,["d"]]], rule.parse(["a","b","d"]))
+  end
+
+  def test_optimize1()
+    rule =
+      token("a") - token("b") - token("c") >> Proc.new{|x| x} |
+      token("a") - token("b") - token("d") >> Proc.new{|x| x}
+    rule = rule.optimize(false)
+    assert_equal(["a","b","c"], rule.parse(["a","b","c"]))
+    assert_equal(["a","b","d"], rule.parse(["a","b","d"]))
+  end
+
   def test_chainl1()
     buff = ["3", "-", "2", "-", "1"]
     rule = chainl(token(/\d+/) >> Proc.new{|x|x[0].to_i}, token("-")){|x|

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: tdp4r
 version: !ruby/object:Gem::Version 
-  version: 1.3.3
-date: 2006-07-19 00:00:00 +09:00
+  version: 1.4.0
+date: 2006-07-22 00:00:00 +09:00
 summary: TDP4R is a top-down parser library that consists of parser combinators and utility functions.
 require_paths: 
 - lib
@@ -36,9 +36,10 @@ files:
 - samples/sample3.rb
 - samples/sample4.rb
 - samples/sample5.rb
-- samples/sample6.rb
-- samples/sample7.rb
+- samples/sample_expr.rb
 - samples/sample_list.rb
+- samples/sample_optimize.rb
+- samples/sample_xml.rb
 - test/test_tdp.rb
 - doc/faq.txt
 - doc/guide.txt

data/samples/sample7.rb

@@ -1,79 +0,0 @@
-require 'tdp'
-require 'tdputils'
-require 'rexml/parsers/pullparser'
-
-class Array
-  def ===(ary)
-    if super(ary)
-      return true
-    end
-    if !ary.is_a?(Array)
-      return false
-    end
-    each_with_index{|v,idx|
-      case ary[idx]
-      when v
-      else
-        return false
-      end
-    }
-    true
-  end
-end
-
-class Hash
-  def ===(h)
-    if super(h)
-      return true
-    end
-    if !h.is_a?(Hash)
-      return false
-    end
-    each{|k,v|
-      case h[k]
-      when v
-      else
-        return false
-      end
-    }
-    true
-  end
-end
-
-translator = TDParser.define{|g|
-  g.xml =
-    token([:start_element, "a", Hash]) -
-      token([:start_element, "b", Hash]) -
-        g.xml*0 -
-      token([:end_element, "b"]) -
-    token([:end_element, "a"]) >> Proc.new{|x|
-      "<foo><bar>#{x[2]}</bar></foo>"
-    } |
-    token([:start_element, String, Hash]) -
-      g.xml*0 -
-    token([:end_element, String]) >> Proc.new{|x|
-      stag = x[0][1].upcase()
-      etag = x[2][1].upcase()
-      "<#{stag}>#{x[1]}</#{etag}>"
-    } |
-    ~token([:end_element, String]) -
-      any() - g.xml >> Proc.new{|x| x[2]}
-
-  def translate(src)
-    xparser = REXML::Parsers::BaseParser.new(src)
-    xml.parse{|g|
-      while(xparser.has_next?)
-        g.yield(xparser.pull())
-      end
-    }
-  end
-}
-
-puts(translator.translate(<<EOS))
-<?xml version="1.0" ?>
-<list>
-  <a><b><c>hoge</c></b></a>
-  <b>b?</b>
-</list>
-EOS
-# => "<LIST><foo><bar><C></C></bar></foo><B></B></LIST>"