kpeg 0.7 → 0.8.0

data/README.md

@@ -5,4 +5,65 @@ KPeg is a simple PEG library for Ruby. It provides an API as well as native gram
 
 KPeg strives to provide a simple, powerful API without being too exotic.
 
-KPeg supports direct left recursion of rules via the [OMeta memoization](http://www.vpri.org/pdf/tr2008003_experimenting.pdf) trick.
+KPeg supports direct left recursion of rules via the [OMeta memoization](http://www.vpri.org/pdf/tr2008003_experimenting.pdf) trick.
+
+## Writing your first grammar
+
+### Setting up your grammar
+
+All grammars start with with the class/module name that will be your parser
+
+    %% name = Example::Parser
+
+After that a block of ruby code can be defined that will be added into the class body of your parser. Attributes that are defines in this block can be accessed within your parser as instance variables
+
+    %% {
+      attr_accessor :something_cool
+    }
+
+### Defining literals
+
+Literals are static declarations of characters or regular expressions designed for reuse in the grammar. These can be constants or variables.
+
+    ALPHA = /[A-Za-z]/
+    DIGIT = /[0-9]/
+    period = "."
+
+Literals can also accept multiple definitions
+
+    vowel = "a" | "e" | "i" | "o" | "u"
+    alpha = /[A-Z]/ | /[a-z]/
+
+### Defining Rules for Values
+
+Before you can start parsing a string you will need to define rules that you will use to accept or reject that string. There are many different types of rules available in kpeg 
+
+The most basic of these rules is a string capture
+  
+    alpha = < /[A-Za-z]/ > { text }
+
+While this looks very much like the ALPHA literal defined above it differs in one important way, the text captured by the rule defined between the < and > symbols will be set as the text variable in block that follows. You can also explicitly define the variable that you would like but only with existing rules or literals.
+    
+    num = /[1-9][0-9]*/
+    sum = < num:n1 "+" num:n2 > { n1 + n2 }
+    
+Additionally blocks can return true or false values based upon an expression within the block. To test if something is true do the following:
+
+    greater_than_10 = < num:n > &{ n > 10 }
+    
+To test for a false value do the following:
+
+    not_greater_than_10 = < num:n > !{ n > 10 }
+    
+Rules can also act like functions and take parameters, an example of this is can be lifted from the [Email List Validator](https://github.com/andrewvc/email_address_validator), where an ascii value is passed in and the character is evaluated against it returning a true if it matches
+    
+    d(num) = <.> &{ text[0] == num }
+    
+
+
+## Projects using kpeg
+
+[Dang](https://github.com/veganstraightedge/dang)
+[Email Address Validator](https://github.com/andrewvc/email_address_validator)
+[Callisto](https://github.com/dwaite/Callisto)
+[Doodle](https://github.com/vito/doodle)

data/lib/kpeg/code_generator.rb

@@ -34,6 +34,59 @@ module KPeg
       @saves = 0
     end
 
+    def output_ast(short, code, description)
+      parser = FormatParser.new description
+
+      # just skip it if it's bad.
+      return unless parser.parse "ast_root"
+
+      name, attrs = parser.result
+
+      code << "    class #{name} < Node\n"
+      code << "      def initialize(#{attrs.join(', ')})\n"
+      attrs.each do |at|
+        code << "        @#{at} = #{at}\n"
+      end
+      code << "      end\n"
+      attrs.each do |at|
+        code << "      attr_reader :#{at}\n"
+      end
+      code << "    end\n"
+
+      [short, name, attrs]
+    end
+
+    def handle_ast(code)
+      output_node = false
+
+      root = @grammar.variables["ast-location"] || "AST"
+
+      methods = []
+
+      @grammar.variables.each do |name, val|
+        if val.index("ast ") == 0
+          unless output_node
+            code << "\n"
+            code << "  module #{root}\n"
+            code << "    class Node; end\n"
+            output_node = true
+          end
+          if m = output_ast(name, code, val[4..-1])
+            methods << m
+          end
+        end
+      end
+
+      if output_node
+        code << "  end\n"
+        methods.each do |short, name, attrs|
+          code << "  def #{short}(#{attrs.join(', ')})\n"
+          code << "    #{root}::#{name}.new(#{attrs.join(', ')})\n"
+          code << "  end\n"
+        end
+      end
+    end
+
     def output_op(code, op)
       case op
       when Dot
@@ -41,7 +94,8 @@ module KPeg
       when LiteralString
         code << "    _tmp = match_string(#{op.string.dump})\n"
       when LiteralRegexp
-        code << "    _tmp = scan(/\\A#{op.regexp}/)\n"
+        lang = op.regexp.kcode.to_s[0,1]
+        code << "    _tmp = scan(/\\A#{op.regexp}/#{lang})\n"
       when CharRange
         ss = save()
         if op.start.bytesize == 1 and op.fin.bytesize == 1
@@ -271,6 +325,8 @@ module KPeg
         code << "\n#{act.action}\n\n"
       end
 
+      handle_ast(code)
+
       fg = @grammar.foreign_grammars
 
       if fg.empty?
@@ -283,6 +339,13 @@ module KPeg
           code << "    @_grammar_#{name} = #{gram}.new(nil)\n"
         end
         code << "  end\n"
+
+        @grammar.foreign_grammars.each do |name, gram|
+          code << "\n"
+          code << "  def invoke_#{name}(*args)\n"
+          code << "    @_grammar_#{name}.external_invoke(self, :_root, *args)\n"
+          code << "  end\n"
+        end
       end
 
       render = GrammarRenderer.new(@grammar)

data/lib/kpeg/compiled_parser.rb

@@ -3,6 +3,11 @@ require 'kpeg/position'
 module KPeg
   class CompiledParser
 
+    # Must be outside the STANDALONE block because a standalone
+    # parser always injects it's own version of this method.
+    def setup_foreign_grammar
+    end
+
     # Leave these markers in! They allow us to generate standalone
     # code automatically!
     #
@@ -18,9 +23,6 @@ module KPeg
       setup_foreign_grammar
     end
 
-    def setup_foreign_grammar
-    end
-
     # This is distinct from setup_parser so that a standalone parser
     # can redefine #initialize and still have access to the proper
     # parser setup code.

data/lib/kpeg/format_parser.rb

@@ -11,9 +11,6 @@ class KPeg::FormatParser
       setup_foreign_grammar
     end
 
-    def setup_foreign_grammar
-    end
-
     # This is distinct from setup_parser so that a standalone parser
     # can redefine #initialize and still have access to the proper
     # parser setup code.
@@ -464,6 +461,32 @@ class KPeg::FormatParser
     return _tmp
   end
 
+  # method = < /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }
+  def _method
+
+    _save = self.pos
+    while true # sequence
+    _text_start = self.pos
+    _tmp = scan(/\A(?-mix:[a-zA-Z_][a-zA-Z0-9_]*)/)
+    if _tmp
+      text = get_text(_text_start)
+    end
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    @result = begin;  text ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save
+    end
+    break
+    end # end sequence
+
+    set_failed_rule :_method unless _tmp
+    return _tmp
+  end
+
   # dbl_escapes = ("\\\"" { '"' } | "\\n" { "\n" } | "\\t" { "\t" } | "\\b" { "\b" } | "\\\\" { "\\" })
   def _dbl_escapes
 
@@ -1239,7 +1262,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # value = (value:v ":" var:n { @g.t(v,n) } | value:v "?" { @g.maybe(v) } | value:v "+" { @g.many(v) } | value:v "*" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | "&" value:v { @g.andp(v) } | "!" value:v { @g.notp(v) } | "(" - expression:o - ")" { o } | "<" - expression:o - ">" { @g.collect(o) } | curly_block | "." { @g.dot } | "@" var:name !(- "=") { @g.invoke(name) } | "^" var:name < nested_paren? > { @g.foreign_invoke("parent", name, text) } | "%" var:gram "." var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- "=") { text.empty? ? @g.ref(name) : @g.invoke(name, text) } | char_range | regexp | string)
+  # value = (value:v ":" var:n { @g.t(v,n) } | value:v "?" { @g.maybe(v) } | value:v "+" { @g.many(v) } | value:v "*" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | "&" value:v { @g.andp(v) } | "!" value:v { @g.notp(v) } | "(" - expression:o - ")" { o } | "<" - expression:o - ">" { @g.collect(o) } | curly_block | "~" method:m < nested_paren? > { @g.action("#{m}#{text}") } | "." { @g.dot } | "@" var:name !(- "=") { @g.invoke(name) } | "^" var:name < nested_paren? > { @g.foreign_invoke("parent", name, text) } | "%" var:gram "." var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- "=") { text.empty? ? @g.ref(name) : @g.invoke(name, text) } | char_range | regexp | string)
   def _value
 
     _save = self.pos
@@ -1503,12 +1526,32 @@ class KPeg::FormatParser
 
     _save10 = self.pos
     while true # sequence
-    _tmp = match_string(".")
+    _tmp = match_string("~")
     unless _tmp
       self.pos = _save10
       break
     end
-    @result = begin;  @g.dot ; end
+    _tmp = apply(:_method)
+    m = @result
+    unless _tmp
+      self.pos = _save10
+      break
+    end
+    _text_start = self.pos
+    _save11 = self.pos
+    _tmp = apply(:_nested_paren)
+    unless _tmp
+      _tmp = true
+      self.pos = _save11
+    end
+    if _tmp
+      text = get_text(_text_start)
+    end
+    unless _tmp
+      self.pos = _save10
+      break
+    end
+    @result = begin;  @g.action("#{m}#{text}") ; end
     _tmp = true
     unless _tmp
       self.pos = _save10
@@ -1519,45 +1562,63 @@ class KPeg::FormatParser
     break if _tmp
     self.pos = _save
 
-    _save11 = self.pos
+    _save12 = self.pos
+    while true # sequence
+    _tmp = match_string(".")
+    unless _tmp
+      self.pos = _save12
+      break
+    end
+    @result = begin;  @g.dot ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save12
+    end
+    break
+    end # end sequence
+
+    break if _tmp
+    self.pos = _save
+
+    _save13 = self.pos
     while true # sequence
     _tmp = match_string("@")
     unless _tmp
-      self.pos = _save11
+      self.pos = _save13
       break
     end
     _tmp = apply(:_var)
     name = @result
     unless _tmp
-      self.pos = _save11
+      self.pos = _save13
       break
     end
-    _save12 = self.pos
+    _save14 = self.pos
 
-    _save13 = self.pos
+    _save15 = self.pos
     while true # sequence
     _tmp = apply(:__hyphen_)
     unless _tmp
-      self.pos = _save13
+      self.pos = _save15
       break
     end
     _tmp = match_string("=")
     unless _tmp
-      self.pos = _save13
+      self.pos = _save15
     end
     break
     end # end sequence
 
     _tmp = _tmp ? nil : true
-    self.pos = _save12
+    self.pos = _save14
     unless _tmp
-      self.pos = _save11
+      self.pos = _save13
       break
     end
     @result = begin;  @g.invoke(name) ; end
     _tmp = true
     unless _tmp
-      self.pos = _save11
+      self.pos = _save13
     end
     break
     end # end sequence
@@ -1565,37 +1626,37 @@ class KPeg::FormatParser
     break if _tmp
     self.pos = _save
 
-    _save14 = self.pos
+    _save16 = self.pos
     while true # sequence
     _tmp = match_string("^")
     unless _tmp
-      self.pos = _save14
+      self.pos = _save16
       break
     end
     _tmp = apply(:_var)
     name = @result
     unless _tmp
-      self.pos = _save14
+      self.pos = _save16
       break
     end
     _text_start = self.pos
-    _save15 = self.pos
+    _save17 = self.pos
     _tmp = apply(:_nested_paren)
     unless _tmp
       _tmp = true
-      self.pos = _save15
+      self.pos = _save17
     end
     if _tmp
       text = get_text(_text_start)
     end
     unless _tmp
-      self.pos = _save14
+      self.pos = _save16
       break
     end
     @result = begin;  @g.foreign_invoke("parent", name, text) ; end
     _tmp = true
     unless _tmp
-      self.pos = _save14
+      self.pos = _save16
     end
     break
     end # end sequence
@@ -1603,48 +1664,48 @@ class KPeg::FormatParser
     break if _tmp
     self.pos = _save
 
-    _save16 = self.pos
+    _save18 = self.pos
     while true # sequence
     _tmp = match_string("%")
     unless _tmp
-      self.pos = _save16
+      self.pos = _save18
       break
     end
     _tmp = apply(:_var)
     gram = @result
     unless _tmp
-      self.pos = _save16
+      self.pos = _save18
       break
     end
     _tmp = match_string(".")
     unless _tmp
-      self.pos = _save16
+      self.pos = _save18
       break
     end
     _tmp = apply(:_var)
     name = @result
     unless _tmp
-      self.pos = _save16
+      self.pos = _save18
       break
     end
     _text_start = self.pos
-    _save17 = self.pos
+    _save19 = self.pos
     _tmp = apply(:_nested_paren)
     unless _tmp
       _tmp = true
-      self.pos = _save17
+      self.pos = _save19
     end
     if _tmp
       text = get_text(_text_start)
     end
     unless _tmp
-      self.pos = _save16
+      self.pos = _save18
       break
     end
     @result = begin;  @g.foreign_invoke(gram, name, text) ; end
     _tmp = true
     unless _tmp
-      self.pos = _save16
+      self.pos = _save18
     end
     break
     end # end sequence
@@ -1652,54 +1713,54 @@ class KPeg::FormatParser
     break if _tmp
     self.pos = _save
 
-    _save18 = self.pos
+    _save20 = self.pos
     while true # sequence
     _tmp = apply(:_var)
     name = @result
     unless _tmp
-      self.pos = _save18
+      self.pos = _save20
       break
     end
     _text_start = self.pos
-    _save19 = self.pos
+    _save21 = self.pos
     _tmp = apply(:_nested_paren)
     unless _tmp
       _tmp = true
-      self.pos = _save19
+      self.pos = _save21
     end
     if _tmp
       text = get_text(_text_start)
     end
     unless _tmp
-      self.pos = _save18
+      self.pos = _save20
       break
     end
-    _save20 = self.pos
+    _save22 = self.pos
 
-    _save21 = self.pos
+    _save23 = self.pos
     while true # sequence
     _tmp = apply(:__hyphen_)
     unless _tmp
-      self.pos = _save21
+      self.pos = _save23
       break
     end
     _tmp = match_string("=")
     unless _tmp
-      self.pos = _save21
+      self.pos = _save23
     end
     break
     end # end sequence
 
     _tmp = _tmp ? nil : true
-    self.pos = _save20
+    self.pos = _save22
     unless _tmp
-      self.pos = _save18
+      self.pos = _save20
       break
     end
     @result = begin;  text.empty? ? @g.ref(name) : @g.invoke(name, text) ; end
     _tmp = true
     unless _tmp
-      self.pos = _save18
+      self.pos = _save20
     end
     break
     end # end sequence
@@ -2009,7 +2070,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
+  # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
   def _statement
 
     _save = self.pos
@@ -2155,7 +2216,7 @@ class KPeg::FormatParser
       break
     end
     _text_start = self.pos
-    _tmp = scan(/\A(?-mix:[::A-Za-z]+)/)
+    _tmp = scan(/\A(?-mix:[::A-Za-z0-9_]+)/)
     if _tmp
       text = get_text(_text_start)
     end
@@ -2402,12 +2463,197 @@ class KPeg::FormatParser
     return _tmp
   end
 
+  # ast_constant = < /[A-Z][A-Za-z0-9_]*/ > { text }
+  def _ast_constant
+
+    _save = self.pos
+    while true # sequence
+    _text_start = self.pos
+    _tmp = scan(/\A(?-mix:[A-Z][A-Za-z0-9_]*)/)
+    if _tmp
+      text = get_text(_text_start)
+    end
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    @result = begin;  text ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save
+    end
+    break
+    end # end sequence
+
+    set_failed_rule :_ast_constant unless _tmp
+    return _tmp
+  end
+
+  # ast_word = < /[A-Za-z_][A-Za-z0-9_]*/ > { text }
+  def _ast_word
+
+    _save = self.pos
+    while true # sequence
+    _text_start = self.pos
+    _tmp = scan(/\A(?-mix:[A-Za-z_][A-Za-z0-9_]*)/)
+    if _tmp
+      text = get_text(_text_start)
+    end
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    @result = begin;  text ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save
+    end
+    break
+    end # end sequence
+
+    set_failed_rule :_ast_word unless _tmp
+    return _tmp
+  end
+
+  # ast_sp = (" " | "\t")*
+  def _ast_sp
+    while true
+
+    _save1 = self.pos
+    while true # choice
+    _tmp = match_string(" ")
+    break if _tmp
+    self.pos = _save1
+    _tmp = match_string("\t")
+    break if _tmp
+    self.pos = _save1
+    break
+    end # end choice
+
+    break unless _tmp
+    end
+    _tmp = true
+    set_failed_rule :_ast_sp unless _tmp
+    return _tmp
+  end
+
+  # ast_words = (ast_words:r ast_sp "," ast_sp ast_word:w { r + [w] } | ast_word:w { [w] })
+  def _ast_words
+
+    _save = self.pos
+    while true # choice
+
+    _save1 = self.pos
+    while true # sequence
+    _tmp = apply(:_ast_words)
+    r = @result
+    unless _tmp
+      self.pos = _save1
+      break
+    end
+    _tmp = apply(:_ast_sp)
+    unless _tmp
+      self.pos = _save1
+      break
+    end
+    _tmp = match_string(",")
+    unless _tmp
+      self.pos = _save1
+      break
+    end
+    _tmp = apply(:_ast_sp)
+    unless _tmp
+      self.pos = _save1
+      break
+    end
+    _tmp = apply(:_ast_word)
+    w = @result
+    unless _tmp
+      self.pos = _save1
+      break
+    end
+    @result = begin;  r + [w] ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save1
+    end
+    break
+    end # end sequence
+
+    break if _tmp
+    self.pos = _save
+
+    _save2 = self.pos
+    while true # sequence
+    _tmp = apply(:_ast_word)
+    w = @result
+    unless _tmp
+      self.pos = _save2
+      break
+    end
+    @result = begin;  [w] ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save2
+    end
+    break
+    end # end sequence
+
+    break if _tmp
+    self.pos = _save
+    break
+    end # end choice
+
+    set_failed_rule :_ast_words unless _tmp
+    return _tmp
+  end
+
+  # ast_root = ast_constant:c "(" ast_words:w ")" { [c, w] }
+  def _ast_root
+
+    _save = self.pos
+    while true # sequence
+    _tmp = apply(:_ast_constant)
+    c = @result
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    _tmp = match_string("(")
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    _tmp = apply(:_ast_words)
+    w = @result
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    _tmp = match_string(")")
+    unless _tmp
+      self.pos = _save
+      break
+    end
+    @result = begin;  [c, w] ; end
+    _tmp = true
+    unless _tmp
+      self.pos = _save
+    end
+    break
+    end # end sequence
+
+    set_failed_rule :_ast_root unless _tmp
+    return _tmp
+  end
+
   Rules = {}
   Rules[:_eol] = rule_info("eol", "\"\\n\"")
   Rules[:_comment] = rule_info("comment", "\"\#\" (!eol .)* eol")
   Rules[:_space] = rule_info("space", "(\" \" | \"\\t\" | eol)")
   Rules[:__hyphen_] = rule_info("-", "(space | comment)*")
   Rules[:_var] = rule_info("var", "< (\"-\" | /[a-zA-Z][\\-_a-zA-Z0-9]*/) > { text }")
+  Rules[:_method] = rule_info("method", "< /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }")
   Rules[:_dbl_escapes] = rule_info("dbl_escapes", "(\"\\\\\\\"\" { '\"' } | \"\\\\n\" { \"\\n\" } | \"\\\\t\" { \"\\t\" } | \"\\\\b\" { \"\\b\" } | \"\\\\\\\\\" { \"\\\\\" })")
   Rules[:_dbl_seq] = rule_info("dbl_seq", "< /[^\\\\\"]+/ > { text }")
   Rules[:_dbl_not_quote] = rule_info("dbl_not_quote", "(dbl_escapes:s | dbl_seq:s)+:ary { ary }")
@@ -2427,14 +2673,19 @@ class KPeg::FormatParser
   Rules[:_curly_block] = rule_info("curly_block", "curly")
   Rules[:_curly] = rule_info("curly", "\"{\" < (/[^{}]+/ | curly)* > \"}\" { @g.action(text) }")
   Rules[:_nested_paren] = rule_info("nested_paren", "\"(\" (/[^()]+/ | nested_paren)* \")\"")
-  Rules[:_value] = rule_info("value", "(value:v \":\" var:n { @g.t(v,n) } | value:v \"?\" { @g.maybe(v) } | value:v \"+\" { @g.many(v) } | value:v \"*\" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | \"&\" value:v { @g.andp(v) } | \"!\" value:v { @g.notp(v) } | \"(\" - expression:o - \")\" { o } | \"<\" - expression:o - \">\" { @g.collect(o) } | curly_block | \".\" { @g.dot } | \"@\" var:name !(- \"=\") { @g.invoke(name) } | \"^\" var:name < nested_paren? > { @g.foreign_invoke(\"parent\", name, text) } | \"%\" var:gram \".\" var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- \"=\") { text.empty? ? @g.ref(name) : @g.invoke(name, text) } | char_range | regexp | string)")
+  Rules[:_value] = rule_info("value", "(value:v \":\" var:n { @g.t(v,n) } | value:v \"?\" { @g.maybe(v) } | value:v \"+\" { @g.many(v) } | value:v \"*\" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | \"&\" value:v { @g.andp(v) } | \"!\" value:v { @g.notp(v) } | \"(\" - expression:o - \")\" { o } | \"<\" - expression:o - \">\" { @g.collect(o) } | curly_block | \"~\" method:m < nested_paren? > { @g.action(\"\#{m}\#{text}\") } | \".\" { @g.dot } | \"@\" var:name !(- \"=\") { @g.invoke(name) } | \"^\" var:name < nested_paren? > { @g.foreign_invoke(\"parent\", name, text) } | \"%\" var:gram \".\" var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- \"=\") { text.empty? ? @g.ref(name) : @g.invoke(name, text) } | char_range | regexp | string)")
   Rules[:_spaces] = rule_info("spaces", "(space | comment)+")
   Rules[:_values] = rule_info("values", "(values:s spaces value:v { @g.seq(s, v) } | value:l spaces value:r { @g.seq(l, r) } | value)")
   Rules[:_choose_cont] = rule_info("choose_cont", "- \"|\" - values:v { v }")
   Rules[:_expression] = rule_info("expression", "(values:v choose_cont+:alts { @g.any(v, *alts) } | values)")
   Rules[:_args] = rule_info("args", "(args:a \",\" - var:n - { a + [n] } | - var:n - { [n] })")
-  Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
+  Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
   Rules[:_statements] = rule_info("statements", "statement (- statements)?")
   Rules[:_eof] = rule_info("eof", "!.")
   Rules[:_root] = rule_info("root", "statements - \"\\n\"? eof")
+  Rules[:_ast_constant] = rule_info("ast_constant", "< /[A-Z][A-Za-z0-9_]*/ > { text }")
+  Rules[:_ast_word] = rule_info("ast_word", "< /[A-Za-z_][A-Za-z0-9_]*/ > { text }")
+  Rules[:_ast_sp] = rule_info("ast_sp", "(\" \" | \"\\t\")*")
+  Rules[:_ast_words] = rule_info("ast_words", "(ast_words:r ast_sp \",\" ast_sp ast_word:w { r + [w] } | ast_word:w { [w] })")
+  Rules[:_ast_root] = rule_info("ast_root", "ast_constant:c \"(\" ast_words:w \")\" { [c, w] }")
 end

data/lib/kpeg/version.rb

@@ -1,3 +1,3 @@
 module KPeg
-  VERSION = "0.7"
+  VERSION = "0.8.0"
 end

data/test/test_kpeg_code_generator.rb

@@ -93,6 +93,36 @@ end
     assert !cg.parse("a")
   end
 
+  def test_reg_unicode
+    gram = KPeg.grammar do |g|
+      g.root = g.reg(/./u)
+    end
+
+    str = <<-STR
+require 'kpeg/compiled_parser'
+
+class Test < KPeg::CompiledParser
+
+  # root = /./u
+  def _root
+    _tmp = scan(/\\A(?-mix:.)/u)
+    set_failed_rule :_root unless _tmp
+    return _tmp
+  end
+
+  Rules = {}
+  Rules[:_root] = rule_info("root", "/./u")
+end
+    STR
+
+    cg = KPeg::CodeGenerator.new "Test", gram
+
+    assert_equal str, cg.output
+
+    assert cg.parse("う")
+    assert cg.parse("a")
+  end
+
   def test_char_range
     gram = KPeg.grammar do |g|
       g.root = g.range("a", "z")
@@ -791,6 +821,10 @@ class Test < KPeg::CompiledParser
     @_grammar_blah = TestKPegCodeGenerator::TestParser.new(nil)
   end
 
+  def invoke_blah(*args)
+    @_grammar_blah.external_invoke(self, :_root, *args)
+  end
+
   # root = %blah.greeting
   def _root
     _tmp = @_grammar_blah.external_invoke(self, :_greeting)
@@ -824,6 +858,10 @@ class Test < KPeg::CompiledParser
     @_grammar_blah = TestKPegCodeGenerator::TestParser.new(nil)
   end
 
+  def invoke_blah(*args)
+    @_grammar_blah.external_invoke(self, :_root, *args)
+  end
+
   # root = %blah.greeting2(1,2)
   def _root
     _tmp = @_grammar_blah.external_invoke(self, :_greeting2, 1,2)
@@ -1304,4 +1342,95 @@ end
     assert cg.parse("hello")
   end
 
+  def test_ast_generation
+    gram = KPeg.grammar do |g|
+      g.root = g.dot
+      g.set_variable "bracket", "ast BracketOperator(receiver, argument)"
+    end
+
+    str = <<-STR
+require 'kpeg/compiled_parser'
+
+class Test < KPeg::CompiledParser
+
+  module AST
+    class Node; end
+    class BracketOperator < Node
+      def initialize(receiver, argument)
+        @receiver = receiver
+        @argument = argument
+      end
+      attr_reader :receiver
+      attr_reader :argument
+    end
+  end
+  def bracket(receiver, argument)
+    AST::BracketOperator.new(receiver, argument)
+  end
+
+  # root = .
+  def _root
+    _tmp = get_byte
+    set_failed_rule :_root unless _tmp
+    return _tmp
+  end
+
+  Rules = {}
+  Rules[:_root] = rule_info("root", ".")
+end
+    STR
+
+    cg = KPeg::CodeGenerator.new "Test", gram
+
+    assert_equal str, cg.output
+
+    assert cg.parse("hello")
+  end
+
+  def test_ast_generation_in_different_location
+    gram = KPeg.grammar do |g|
+      g.root = g.dot
+      g.set_variable "bracket", "ast BracketOperator(receiver, argument)"
+      g.set_variable "ast-location", "MegaAST"
+    end
+
+    str = <<-STR
+require 'kpeg/compiled_parser'
+
+class Test < KPeg::CompiledParser
+
+  module MegaAST
+    class Node; end
+    class BracketOperator < Node
+      def initialize(receiver, argument)
+        @receiver = receiver
+        @argument = argument
+      end
+      attr_reader :receiver
+      attr_reader :argument
+    end
+  end
+  def bracket(receiver, argument)
+    MegaAST::BracketOperator.new(receiver, argument)
+  end
+
+  # root = .
+  def _root
+    _tmp = get_byte
+    set_failed_rule :_root unless _tmp
+    return _tmp
+  end
+
+  Rules = {}
+  Rules[:_root] = rule_info("root", ".")
+end
+    STR
+
+    cg = KPeg::CodeGenerator.new "Test", gram
+
+    assert_equal str, cg.output
+
+    assert cg.parse("hello")
+  end
+
 end

data/test/test_kpeg_format.rb

@@ -109,6 +109,16 @@ b(p) = x
     assert_equal "OtherGrammar", gram.foreign_grammars["blah"]
   end
 
+  def test_add_foreign_grammar_with_numbers
+    gram = match "%blah = Thing1::OtherGrammar"
+    assert_equal "Thing1::OtherGrammar", gram.foreign_grammars["blah"]
+  end
+
+  def test_add_foreign_grammar_with_undescore
+    gram = match "%blah = Other_Grammar"
+    assert_equal "Other_Grammar", gram.foreign_grammars["blah"]
+  end
+
   def test_invoke_parent_rule
     assert_rule G.foreign_invoke("parent", "letters"),
                 match("a=^letters"), "a"
@@ -305,6 +315,16 @@ Value   = NUMBER:i                      { i }
     assert_rule G.seq(:b, :c, G.action(" b + { c + d } ")), m
   end
 
+  def test_action_send
+    m = match 'a=b c ~d'
+    assert_rule G.seq(:b, :c, G.action("d")), m
+  end
+
+  def test_action_send_with_args
+    m = match 'a=b c ~d(b,c)'
+    assert_rule G.seq(:b, :c, G.action("d(b,c)")), m
+  end
+
   def test_collect
     m = match 'a = < b c >'
     assert_rule G.collect(G.seq(:b, :c)), m

metadata

@@ -1,11 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: kpeg
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 63
+  prerelease: 
   segments: 
   - 0
-  - 7
-  version: "0.7"
+  - 8
+  - 0
+  version: 0.8.0
 platform: ruby
 authors: 
 - Evan Phoenix
@@ -13,16 +15,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-15 00:00:00 -07:00
+date: 2011-04-06 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rake
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -57,6 +61,13 @@ files:
 - Rakefile
 - kpeg.gemspec
 - Gemfile
+- test/test_file_parser_roundtrip.rb
+- test/test_gen_calc.rb
+- test/test_kpeg.rb
+- test/test_kpeg_code_generator.rb
+- test/test_kpeg_compiled_parser.rb
+- test/test_kpeg_format.rb
+- test/test_kpeg_grammar_renderer.rb
 has_rdoc: true
 homepage: https://github.com/evanphx/kpeg
 licenses: []
@@ -67,23 +78,27 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Peg-based Code Generator