kpeg 0.8.5 → 0.9.0

data/lib/kpeg/format_parser.kpeg

@@ -0,0 +1,129 @@
+%% name = KPeg::FormatParser
+
+%% {
+    require 'kpeg/grammar'
+
+    def initialize(str, debug=false)
+      setup_parser(str, debug)
+      @g = KPeg::Grammar.new
+    end
+
+    attr_reader :g
+    alias_method :grammar, :g
+}
+
+
+             eol = "\n"
+     eof_comment = "#" (!eof .)*
+
+         comment = "#" (!eol .)* eol
+           space = " " | "\t" | eol
+               - = (space | comment)*
+          kleene = "*"
+
+                   # Allow - by itself, but not at the beginning
+             var = < "-" | /[a-zA-Z][\-_a-zA-Z0-9]*/ > { text }
+          method = < /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }
+
+     dbl_escapes = "n" { "\n" }
+                 | "s" { " " }
+                 | "r" { "\r" }
+                 | "t" { "\t" }
+                 | "v" { "\v" }
+                 | "f" { "\f" }
+                 | "b" { "\b" }
+                 | "a" { "\a" }
+                 | "e" { "\e" }
+                 | "\\" { "\\" }
+                 | "\"" { "\"" }
+                 | num_escapes
+                 | < . > { text }
+     num_escapes = < /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") }
+                 | "x" < /[0-9a-fA-F]{2}/ > { [text.to_i(16)].pack("U") }
+         dbl_seq = < /[^\\"]+/ > { text }
+   dbl_not_quote = ("\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) }
+      dbl_string = "\"" dbl_not_quote:s "\"" { @g.str(s.join) }
+sgl_escape_quote = "\\'" { "'" }
+         sgl_seq = < /[^']/ > { text }
+   sgl_not_quote = (sgl_escape_quote | sgl_seq)*:segs { Array(segs) }
+      sgl_string = "'" sgl_not_quote:s "'" { @g.str(s.join) }
+          string = dbl_string
+                 | sgl_string
+
+       not_slash = < ("\\/" | /[^\/]/)+ > { text }
+     regexp_opts = < [a-z]* > { text }
+          regexp = "/" not_slash:body "/" regexp_opts:opts
+                   { @g.reg body, opts }
+
+            char = < /[a-zA-Z0-9]/ > { text }
+      char_range = "[" char:l "-" char:r "]" { @g.range(l,r) }
+
+       range_num = < /[1-9][0-9]*/ > { text }
+      range_elem = < range_num|kleene > { text }
+      mult_range = "[" - range_elem:l - "," - range_elem:r - "]"
+                   { [l == "*" ? nil : l.to_i, r == "*" ? nil : r.to_i] }
+                 | "[" - range_num:e - "]" { [e.to_i, e.to_i] }
+
+     curly_block = curly
+           curly = "{" < (/[^{}"']+/ | string | curly)* > "}" { @g.action(text) }
+    nested_paren = "(" (/[^()"']+/ | string | nested_paren)* ")"
+
+           value = value:v ":" var:n { @g.t(v,n) }
+                 | value:v "?" { @g.maybe(v) }
+                 | value:v "+" { @g.many(v) }
+                 | value:v "*" { @g.kleene(v) }
+                 | value:v mult_range:r { @g.multiple(v, *r) }
+                 | "&" value:v { @g.andp(v) }
+                 | "!" value:v { @g.notp(v) }
+                 | "(" - expression:o - ")" { o }
+                 | "@<" - expression:o - ">" { @g.bounds(o) }
+                 | "<" - expression:o - ">" { @g.collect(o) }
+                 | curly_block
+                 | "~" method:m < nested_paren? >
+                   { @g.action("#{m}#{text}") }
+                 | "." { @g.dot }
+                 | "@" var:name < nested_paren? > !(- "=")
+                   { @g.invoke(name, text.empty? ? nil : text) }
+                 | "^" var:name < nested_paren? >
+                   { @g.foreign_invoke("parent", name, text) }
+                 | "%" var:gram "." var:name < nested_paren? >
+                   { @g.foreign_invoke(gram, name, text) }
+                 | var:name < nested_paren? > !(- "=")
+                   { @g.ref(name, nil, text.empty? ? nil : text) }
+                 | char_range
+                 | regexp
+                 | string
+
+          spaces = (space | comment)+
+          values = values:s spaces value:v { @g.seq(s, v) }
+                 | value:l spaces value:r  { @g.seq(l, r) }
+                 | value
+     choose_cont = - "|" - values:v { v }
+      expression = values:v choose_cont+:alts { @g.any(v, *alts) }
+                 | values
+            args = args:a "," - var:n - { a + [n] }
+                 | - var:n - { [n] }
+       statement = - var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) }
+                 | - var:v - "=" - expression:o { @g.set(v, o) }
+                 | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ >
+                   { @g.add_foreign_grammar(name, text) }
+                 | - "%%" - curly:act { @g.add_setup act }
+                 | - "%%" - var:name - curly:act { @g.add_directive name, act }
+                 | - "%%" - var:name - "=" - < (!"\n" .)+ >
+                   { @g.set_variable(name, text) }
+      statements = statement (- statements)?
+             eof = !.
+            root = statements - eof_comment? eof
+
+# These are a seperate set of rules used to parse an ast declaration
+
+    ast_constant = < /[A-Z][A-Za-z0-9_]*/ > { text }
+        ast_word = < /[A-Za-z_][A-Za-z0-9_]*/ > { text }
+
+          ast_sp = (" " | "\t")*
+
+       ast_words = ast_words:r ast_sp "," ast_sp ast_word:w { r + [w] }
+                 | ast_word:w { [w] }
+
+        ast_root = ast_constant:c "(" ast_words:w ")" { [c, w] }
+                 | ast_constant:c "()"? { [c, []] }

data/lib/kpeg/format_parser.rb

@@ -186,41 +186,37 @@ class KPeg::FormatParser
     end
 
     def parse(rule=nil)
+      # We invoke the rules indirectly via apply
+      # instead of by just calling them as methods because
+      # if the rules use left recursion, apply needs to
+      # manage that.
+
       if !rule
-        _root ? true : false
+        apply(:_root)
       else
-        # This is not shared with code_generator.rb so this can be standalone
         method = rule.gsub("-","_hyphen_")
-        __send__("_#{method}") ? true : false
+        apply :"_#{method}"
       end
     end
 
-    class LeftRecursive
-      def initialize(detected=false)
-        @detected = detected
-      end
-
-      attr_accessor :detected
-    end
-
     class MemoEntry
       def initialize(ans, pos)
         @ans = ans
         @pos = pos
-        @uses = 1
         @result = nil
+        @set = false
+        @left_rec = false
       end
 
-      attr_reader :ans, :pos, :uses, :result
-
-      def inc!
-        @uses += 1
-      end
+      attr_reader :ans, :pos, :result, :set
+      attr_accessor :left_rec
 
       def move!(ans, pos, result)
         @ans = ans
         @pos = pos
         @result = result
+        @set = true
+        @left_rec = false
       end
     end
 
@@ -248,12 +244,10 @@ class KPeg::FormatParser
     def apply_with_args(rule, *args)
       memo_key = [rule, args]
       if m = @memoizations[memo_key][@pos]
-        m.inc!
-
         prev = @pos
         @pos = m.pos
-        if m.ans.kind_of? LeftRecursive
-          m.ans.detected = true
+        if !m.set
+          m.left_rec = true
           return nil
         end
 
@@ -261,18 +255,19 @@ class KPeg::FormatParser
 
         return m.ans
       else
-        lr = LeftRecursive.new(false)
-        m = MemoEntry.new(lr, @pos)
+        m = MemoEntry.new(nil, @pos)
         @memoizations[memo_key][@pos] = m
         start_pos = @pos
 
         ans = __send__ rule, *args
 
+        lr = m.left_rec
+
         m.move! ans, @pos, @result
 
         # Don't bother trying to grow the left recursion
         # if it's failing straight away (thus there is no seed)
-        if ans and lr.detected
+        if ans and lr
           return grow_lr(rule, args, start_pos, m)
         else
           return ans
@@ -284,12 +279,10 @@ class KPeg::FormatParser
 
     def apply(rule)
       if m = @memoizations[rule][@pos]
-        m.inc!
-
         prev = @pos
         @pos = m.pos
-        if m.ans.kind_of? LeftRecursive
-          m.ans.detected = true
+        if !m.set
+          m.left_rec = true
           return nil
         end
 
@@ -297,18 +290,19 @@ class KPeg::FormatParser
 
         return m.ans
       else
-        lr = LeftRecursive.new(false)
-        m = MemoEntry.new(lr, @pos)
+        m = MemoEntry.new(nil, @pos)
         @memoizations[rule][@pos] = m
         start_pos = @pos
 
         ans = __send__ rule
 
+        lr = m.left_rec
+
         m.move! ans, @pos, @result
 
         # Don't bother trying to grow the left recursion
         # if it's failing straight away (thus there is no seed)
-        if ans and lr.detected
+        if ans and lr
           return grow_lr(rule, nil, start_pos, m)
         else
           return ans
@@ -2474,7 +2468,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
+  # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - curly:act { @g.add_directive name, act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
   def _statement
 
     _save = self.pos
@@ -2701,32 +2695,77 @@ class KPeg::FormatParser
           self.pos = _save5
           break
         end
-        _tmp = match_string("=")
+        _tmp = apply(:_curly)
+        act = @result
         unless _tmp
           self.pos = _save5
           break
         end
-        _tmp = apply(:__hyphen_)
+        @result = begin;  @g.add_directive name, act ; end
+        _tmp = true
         unless _tmp
           self.pos = _save5
+        end
+        break
+      end # end sequence
+
+      break if _tmp
+      self.pos = _save
+
+      _save6 = self.pos
+      while true # sequence
+        _tmp = apply(:__hyphen_)
+        unless _tmp
+          self.pos = _save6
+          break
+        end
+        _tmp = match_string("%%")
+        unless _tmp
+          self.pos = _save6
+          break
+        end
+        _tmp = apply(:__hyphen_)
+        unless _tmp
+          self.pos = _save6
+          break
+        end
+        _tmp = apply(:_var)
+        name = @result
+        unless _tmp
+          self.pos = _save6
+          break
+        end
+        _tmp = apply(:__hyphen_)
+        unless _tmp
+          self.pos = _save6
+          break
+        end
+        _tmp = match_string("=")
+        unless _tmp
+          self.pos = _save6
+          break
+        end
+        _tmp = apply(:__hyphen_)
+        unless _tmp
+          self.pos = _save6
           break
         end
         _text_start = self.pos
-        _save6 = self.pos
-
         _save7 = self.pos
+
+        _save8 = self.pos
         while true # sequence
-          _save8 = self.pos
+          _save9 = self.pos
           _tmp = match_string("\n")
           _tmp = _tmp ? nil : true
-          self.pos = _save8
+          self.pos = _save9
           unless _tmp
-            self.pos = _save7
+            self.pos = _save8
             break
           end
           _tmp = get_byte
           unless _tmp
-            self.pos = _save7
+            self.pos = _save8
           end
           break
         end # end sequence
@@ -2734,19 +2773,19 @@ class KPeg::FormatParser
         if _tmp
           while true
 
-            _save9 = self.pos
+            _save10 = self.pos
             while true # sequence
-              _save10 = self.pos
+              _save11 = self.pos
               _tmp = match_string("\n")
               _tmp = _tmp ? nil : true
-              self.pos = _save10
+              self.pos = _save11
               unless _tmp
-                self.pos = _save9
+                self.pos = _save10
                 break
               end
               _tmp = get_byte
               unless _tmp
-                self.pos = _save9
+                self.pos = _save10
               end
               break
             end # end sequence
@@ -2755,19 +2794,19 @@ class KPeg::FormatParser
           end
           _tmp = true
         else
-          self.pos = _save6
+          self.pos = _save7
         end
         if _tmp
           text = get_text(_text_start)
         end
         unless _tmp
-          self.pos = _save5
+          self.pos = _save6
           break
         end
         @result = begin;  @g.set_variable(name, text) ; end
         _tmp = true
         unless _tmp
-          self.pos = _save5
+          self.pos = _save6
         end
         break
       end # end sequence
@@ -3124,7 +3163,7 @@ class KPeg::FormatParser
   Rules[:_choose_cont] = rule_info("choose_cont", "- \"|\" - values:v { v }")
   Rules[:_expression] = rule_info("expression", "(values:v choose_cont+:alts { @g.any(v, *alts) } | values)")
   Rules[:_args] = rule_info("args", "(args:a \",\" - var:n - { a + [n] } | - var:n - { [n] })")
-  Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
+  Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - curly:act { @g.add_directive name, act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
   Rules[:_statements] = rule_info("statements", "statement (- statements)?")
   Rules[:_eof] = rule_info("eof", "!.")
   Rules[:_root] = rule_info("root", "statements - eof_comment? eof")

data/lib/kpeg/grammar.rb

@@ -634,6 +634,7 @@ module KPeg
 
   class Grammar
     def initialize
+      @directives = {}
       @rules = {}
       @rule_order = []
       @setup_actions = []
@@ -641,9 +642,18 @@ module KPeg
       @variables = {}
     end
 
+    attr_reader :directives
     attr_reader :rules, :rule_order, :setup_actions, :foreign_grammars
     attr_reader :variables
 
+    def add_directive(name, body)
+      if @directives.include? name
+        warn "directive #{name.inspect} appears more than once"
+      end
+
+      @directives[name] = body
+    end
+
     def add_setup(act)
       @setup_actions << act
     end

data/lib/kpeg/string_escape.kpeg

@@ -0,0 +1,20 @@
+%% name = KPeg::StringEscape
+
+%% {
+  attr_reader :text
+}
+
+segment = < /[\w ]+/ > { text } # Don't use \s because that matchs \n
+        | "\\" { "\\\\" }
+        | "\n" { "\\n" }
+        | "\t" { "\\t" }
+        | "\b" { "\\b" }
+        | "\"" { "\\\"" }
+        | < . > { text }
+
+root = segment*:s { @text = s.join }
+
+embed_seg = "#" { "\\#" }
+          | segment
+
+embed = embed_seg*:s { @text = s.join }