tdp4r 1.3.3

data/lib/tdputils.rb

@@ -0,0 +1,89 @@
+require 'tdp'
+
+module TDPUtils
+  class Token
+    attr_accessor :kind, :value
+    def initialize(kind, value)
+      @kind = kind
+      @value = value
+    end
+
+    def ==(other)
+      (other.class == self.class) &&
+      (@kind == other.kind) &&
+      (@value == other.value)
+    end
+
+    def ===(other)
+      super(other) || (@kind == other)
+    end
+
+    def =~(other)
+      @kind == other
+    end
+  end
+
+  class BasicStringTokenizer
+    def self.[](rule, ignore=nil)
+      self.new(rule, ignore)
+    end
+
+    def initialize(rule, ignore=nil)
+      require("strscan")
+      @rule = rule
+      @scan_pattern = Regexp.new(@rule.keys.join("|"))
+      @ignore_pattern = ignore
+    end
+
+    def generate(str)
+      scanner = StringScanner.new(str)
+      TDParser::TokenGenerator.new{|x|
+        while(!scanner.empty?)
+          if (@ignore_pattern)
+            while(scanner.scan(@ignore_pattern))
+            end
+          end
+          sstr = scanner.scan(@scan_pattern)
+          if (sstr)
+            @rule.each{|reg,kind|
+              if (reg =~ sstr)
+                x.yield(Token.new(kind, sstr))
+                yielded = true
+                break
+              end
+            }
+          else
+            c = scanner.scan(/./)
+            x.yield(c)
+          end
+        end
+      }
+    end
+  end
+
+  class StringTokenizer < BasicStringTokenizer
+    def initialize(rule, ignore=nil)
+      super(rule, ignore || /\s+/)
+    end
+  end
+
+  class WaitingTokenGenerator < TDParser::TokenGenerator
+    def initialize(*args)
+      super(*args)
+      @terminated = false
+    end
+
+    def terminate()
+      @terminated = true
+    end
+
+    def shift()
+      if (@terminated)
+        return nil
+      end
+      while(empty?())
+      end
+      super()
+    end
+  end
+end

data/samples/sample1.rb

@@ -0,0 +1,32 @@
+# -*- ruby -*-
+# adder-substractor
+
+require 'tdp'
+
+class MyParser
+  include TDParser
+
+  def expr
+    token(/\d+/) - ((token("+")|token("-")) - token(/\d+/))*0 >> proc{|x|
+      n = x[0].to_i
+      x[1].inject(n){|acc,i|
+        case i[0]
+        when "-"
+          acc - i[1].to_i
+        when "+"
+          acc + i[1].to_i
+        end
+      }
+    }
+  end
+
+  def parse(str)
+    tokens = str.split(/(?:\s+)|([\+\-\*\/])/).select{|x| x != ""}
+    expr.parse(tokens)
+  end
+end
+
+parser = MyParser.new
+puts("1+10 = " + parser.parse("1+10").to_s())
+puts("2-1-20 = " + parser.parse("2 - 1 - 20").to_s())
+puts("1+2-3 = " + parser.parse("1 + 2 - 3").to_s())

data/samples/sample2.rb

@@ -0,0 +1,55 @@
+# -*- ruby -*-
+# parsing four arithmetic expressions with tdputils.
+
+require 'tdp'
+require 'tdputils'
+
+class MyParser
+  include TDParser
+  include TDPUtils
+
+  def expr1
+    rule(:expr2) - ((token("+")|token("-")) - rule(:expr2))*0 >> proc{|x|
+      x[1].inject(x[0]){|n,y|
+        case y[0]
+        when "+"
+          n + y[1]
+        when "-"
+          n - y[1]
+        end
+      }
+    }
+  end
+
+  def expr2
+    rule(:prim) - ((token("*")|token("/")) - rule(:prim))*0 >> proc{|x|
+      x[1].inject(x[0]){|n, y|
+        case y[0]
+        when "*"
+          n * y[1]
+        when "/"
+          n / y[1]
+        end
+      }
+    }
+  end
+
+  def prim
+    token(:int) >> proc{|x| x[0].value.to_i } |
+    token("(") - rule(:expr1) - token(")") >> proc{|x| x[1] }
+  end
+
+  def parse(str)
+    tokenizer = StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+parser = MyParser.new
+puts("1+10 = " + parser.parse("1+10").to_s())
+puts("2-1*20+18 = " + parser.parse("2 - 1 * 20 + 18").to_s())
+puts("2-(1-20) = " + parser.parse("2 - (1 - 20)").to_s())
+puts("1+2-3 = " + parser.parse("1 + 2 - 3").to_s())

data/samples/sample3.rb

@@ -0,0 +1,57 @@
+# -*- ruby -*-
+# parsing four arithmetic expressions with tdputils.
+
+require 'tdp'
+require 'tdputils'
+
+class MyParser
+  include TDParser
+  include TDPUtils
+
+  def expr1
+    n = nil;
+    (rule(:expr2) >> proc{|x| n = x[0] }) -
+    ((token("+")|token("-")) - rule(:expr2) >> proc{|x|
+      case x[0]
+      when "+"
+        n += x[1]
+      when "-"
+        n -= x[1]
+      end
+      n
+    })*0 >> proc{ n }
+  end
+
+  def expr2
+    n = nil;
+    (rule(:prim) >> proc{|x| n = x[0] }) -
+    ((token("*")|token("/")) - rule(:prim) >> proc{|x|
+      case x[0]
+      when "*"
+        n *= x[1]
+      when "/"
+        n /= x[1]
+      end
+      n
+    })*0 >> proc{ n }
+  end
+
+  def prim
+    token(:int) >> proc{|x| x[0].value.to_i } |
+    token("(") - rule(:expr1) - token(")") >> proc{|x| x[1] }
+  end
+
+  def parse(str)
+    tokenizer = StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+parser = MyParser.new
+puts("1+10 = " + parser.parse("1+10").to_s())
+puts("2-1*20+18 = " + parser.parse("2 - 1 * 20 + 18").to_s())
+puts("2-(1-20) = " + parser.parse("2 - (1 - 20)").to_s())
+puts("1+2-3 = " + parser.parse("1 + 2 - 3").to_s())

data/samples/sample4.rb

@@ -0,0 +1,75 @@
+# -*- ruby -*-
+# caching constructed grammars
+
+require 'tdp'
+require 'tdputils'
+
+class MyParser
+  include TDParser
+  include TDPUtils
+
+  def expr1
+    rule(:expr2) - ((token("+")|token("-")) - rule(:expr2))*0 >> proc{|x|
+      x[1].inject(x[0]){|n,y|
+        case y[0]
+        when "+"
+          n + y[1]
+        when "-"
+          n - y[1]
+        end
+      }
+    }
+  end
+
+  def expr2
+    rule(:prim) - ((token("*")|token("/")) - rule(:prim))*0 >> proc{|x|
+      n = x[0]
+      x[1].inject(x[0]){|n,y|
+        case y[0]
+        when "*"
+          n * y[1]
+        when "/"
+          n / y[1]
+        end
+      }
+    }
+  end
+
+  def prim
+    token(:int) >> proc{|x| x[0].value.to_i } |
+    token("(") - rule(:expr1) - token(")") >> proc{|x| x[1] }
+  end
+
+  def parse(str)
+    tokenizer = StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+end
+
+class FastParser < MyParser
+  def expr1
+    @expr1 ||= super()
+  end
+
+  def expr2
+    @expr2 ||= super()
+  end
+
+  def prim
+    @prim  ||= super()
+  end
+
+  def parse(str)
+    tokens = str.split(/(?:\s+)|([\(\)\+\-\*\/])/).select{|x| x != ""}
+    expr1.parse(tokens)
+  end
+end
+
+parser = MyParser.new
+puts("1+10 = " + parser.parse("1+10").to_s())
+puts("2-1*20+18 = " + parser.parse("2 - 1 * 20 + 18").to_s())
+puts("2-(1-20) = " + parser.parse("2 - (1 - 20)").to_s())
+puts("1+2-3 = " + parser.parse("1 + 2 - 3").to_s())

data/samples/sample5.rb

@@ -0,0 +1,53 @@
+# -*- ruby -*-
+# writing grammars in the substitution style.
+
+require 'tdp'
+require 'tdputils'
+
+parser = TDParser.define{|g|
+  g.plus = "+"
+  g.minus = "-"
+  g.mult = "*"
+  g.div = "/"
+
+  g.expr1 =
+    g.expr2 - ((g.plus|g.minus) - g.expr2)*0 >> proc{|x|
+      x[1].inject(x[0]){|n,y|
+        case y[0]
+        when "+"
+          n + y[1]
+        when "-"
+          n - y[1]
+        end
+      }
+    }
+
+  g.expr2 =
+    g.prim - ((g.mult|g.div) - g.prim)*0 >> proc{|x|
+      x[1].inject(x[0]){|n,y|
+        case y[0]
+        when "*"
+          n * y[1]
+        when "/"
+          n / y[1]
+        end
+      }
+    }
+
+  g.prim =
+    g.token(:int) >> proc{|x| x[0].value.to_i } |
+    g.token("(") - g.expr1 - g.token(")") >> proc{|x| x[1] }
+
+  def parse(str)
+    tokenizer = TDPUtils::StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+}
+
+puts("1+10 = " + parser.parse("1+10").to_s())
+puts("2-1*20+18 = " + parser.parse("2 - 1 * 20 + 18").to_s())
+puts("2-(1-20) = " + parser.parse("2 - (1 - 20)").to_s())
+puts("1+2-3 = " + parser.parse("1 + 2 - 3").to_s())

data/samples/sample6.rb

@@ -0,0 +1,44 @@
+# -*- ruby -*-
+# writing grammars using chainl().
+
+require 'tdp'
+require 'tdputils'
+
+parser = TDParser.define{|g|
+  g.plus = "+"
+  g.minus = "-"
+  g.mult = "*"
+  g.div = "/"
+
+  g.expr1 =
+    chainl(prim, mult|div, plus|minus){|x|
+      case x[1]
+      when "+"
+        x[0] + x[2]
+      when "-"
+        x[0] - x[2]
+      when "*"
+        x[0] * x[2]
+      when "/"
+        x[0] / x[2]
+      end
+    }
+
+  g.prim =
+    token(:int) >> proc{|x| x[0].value.to_i } |
+    token("(") - expr1 - token(")") >> proc{|x| x[1] }
+
+  def parse(str)
+    tokenizer = TDPUtils::StringTokenizer[
+      /\d+(?!\.\d)/ => :int,
+      /\d+\.\d+/ => :real,
+    ]
+    expr1.parse(tokenizer.generate(str))
+  end
+}
+
+puts("1 = " + parser.parse("1").to_s())
+puts("1+10 = " + parser.parse("1+10").to_s())
+puts("2-1*20+18 = " + parser.parse("2 - 1 * 20 + 18").to_s())
+puts("2-(1-20) = " + parser.parse("2 - (1 - 20)").to_s())
+puts("1+2-3 = " + parser.parse("1 + 2 - 3").to_s())

data/samples/sample7.rb

@@ -0,0 +1,79 @@
+require 'tdp'
+require 'tdputils'
+require 'rexml/parsers/pullparser'
+
+class Array
+  def ===(ary)
+    if super(ary)
+      return true
+    end
+    if !ary.is_a?(Array)
+      return false
+    end
+    each_with_index{|v,idx|
+      case ary[idx]
+      when v
+      else
+        return false
+      end
+    }
+    true
+  end
+end
+
+class Hash
+  def ===(h)
+    if super(h)
+      return true
+    end
+    if !h.is_a?(Hash)
+      return false
+    end
+    each{|k,v|
+      case h[k]
+      when v
+      else
+        return false
+      end
+    }
+    true
+  end
+end
+
+translator = TDParser.define{|g|
+  g.xml =
+    token([:start_element, "a", Hash]) -
+      token([:start_element, "b", Hash]) -
+        g.xml*0 -
+      token([:end_element, "b"]) -
+    token([:end_element, "a"]) >> Proc.new{|x|
+      "<foo><bar>#{x[2]}</bar></foo>"
+    } |
+    token([:start_element, String, Hash]) -
+      g.xml*0 -
+    token([:end_element, String]) >> Proc.new{|x|
+      stag = x[0][1].upcase()
+      etag = x[2][1].upcase()
+      "<#{stag}>#{x[1]}</#{etag}>"
+    } |
+    ~token([:end_element, String]) -
+      any() - g.xml >> Proc.new{|x| x[2]}
+
+  def translate(src)
+    xparser = REXML::Parsers::BaseParser.new(src)
+    xml.parse{|g|
+      while(xparser.has_next?)
+        g.yield(xparser.pull())
+      end
+    }
+  end
+}
+
+puts(translator.translate(<<EOS))
+<?xml version="1.0" ?>
+<list>
+  <a><b><c>hoge</c></b></a>
+  <b>b?</b>
+</list>
+EOS
+# => "<LIST><foo><bar><C></C></bar></foo><B></B></LIST>"

data/samples/sample_list.rb

@@ -0,0 +1,21 @@
+require 'tdp'
+
+parser = TDParser.define{|g|
+  g.lp = "("
+  g.rp = ")"
+  g.str = /\w+/
+
+  # Note that "g.elem*1" is a iteration of a sequence that consists
+  # of only "g.elem", but it is not a iteration of "g.elem".
+  g.list = g.lp - g.elem*1 - g.rp >> proc{|x| x[1].collect{|y| y[0]} }
+  g.elem = (g.str | g.list) >> proc{|x| x[0]}
+
+  def parse(str)
+    buff = str.split(/\s+|([\(\)])/).select{|s| s.size() > 0}
+    list.parse(buff)
+  end
+}
+
+list = "(a (b c d) (e f g))"
+r = parser.parse(list)
+p r