kpeg 0.9.0 → 1.2.0

data/lib/kpeg/format_parser.rb

@@ -1,29 +1,24 @@
+require 'kpeg/grammar'
 class KPeg::FormatParser
-# STANDALONE START
+  # :stopdoc:
+
+    # Prepares for parsing +str+.  If you define a custom initialize you must
+    # call this method before #parse
     def setup_parser(str, debug=false)
-      @string = str
-      @pos = 0
+      set_string str, 0
       @memoizations = Hash.new { |h,k| h[k] = {} }
       @result = nil
       @failed_rule = nil
       @failing_rule_offset = -1
+      @line_offsets = nil
 
       setup_foreign_grammar
     end
 
-    # This is distinct from setup_parser so that a standalone parser
-    # can redefine #initialize and still have access to the proper
-    # parser setup code.
-    #
-    def initialize(str, debug=false)
-      setup_parser(str, debug)
-    end
-
     attr_reader :string
     attr_reader :failing_rule_offset
     attr_accessor :result, :pos
 
-    # STANDALONE START
     def current_column(target=pos)
       if c = string.rindex("\n", target-1)
         return target - c - 1
@@ -32,17 +27,33 @@ class KPeg::FormatParser
       target + 1
     end
 
-    def current_line(target=pos)
-      cur_offset = 0
-      cur_line = 0
+    if [].respond_to? :bsearch_index
+      def current_line(target=pos)
+        unless @line_offsets
+          @line_offsets = [-1]
+          total = 0
+          string.each_line do |line|
+            @line_offsets << total
+            total += line.size
+          end
+          @line_offsets << total
+        end
 
-      string.each_line do |line|
-        cur_line += 1
-        cur_offset += line.size
-        return cur_line if cur_offset >= target
+        @line_offsets.bsearch_index {|x| x >= target } || -1
       end
+    else
+      def current_line(target=pos)
+        cur_offset = 0
+        cur_line = 0
 
-      -1
+        string.each_line do |line|
+          cur_line += 1
+          cur_offset += line.size
+          return cur_line if cur_offset >= target
+        end
+
+        -1
+      end
     end
 
     def lines
@@ -51,12 +62,19 @@ class KPeg::FormatParser
       lines
     end
 
-    #
+
 
     def get_text(start)
       @string[start..@pos-1]
     end
 
+    # Sets the string and current parsing position for the parser.
+    def set_string string, pos
+      @string = string
+      @string_size = string ? string.size : 0
+      @pos = pos
+    end
+
     def show_pos
       width = 10
       if @pos < width
@@ -154,28 +172,27 @@ class KPeg::FormatParser
     end
 
     def scan(reg)
-      if m = reg.match(@string[@pos..-1])
-        width = m.end(0)
-        @pos += width
+      if m = reg.match(@string, @pos)
+        @pos = m.end(0)
         return true
       end
 
       return nil
     end
 
-    if "".respond_to? :getbyte
+    if "".respond_to? :ord
       def get_byte
-        if @pos >= @string.size
+        if @pos >= @string_size
           return nil
         end
 
-        s = @string.getbyte @pos
+        s = @string[@pos].ord
         @pos += 1
         s
       end
     else
       def get_byte
-        if @pos >= @string.size
+        if @pos >= @string_size
           return nil
         end
 
@@ -224,8 +241,7 @@ class KPeg::FormatParser
       old_pos = @pos
       old_string = @string
 
-      @pos = other.pos
-      @string = other.string
+      set_string other.string, other.pos
 
       begin
         if val = __send__(rule, *args)
@@ -236,15 +252,13 @@ class KPeg::FormatParser
         end
         val
       ensure
-        @pos = old_pos
-        @string = old_string
+        set_string old_string, old_pos
       end
     end
 
     def apply_with_args(rule, *args)
       memo_key = [rule, args]
       if m = @memoizations[memo_key][@pos]
-        prev = @pos
         @pos = m.pos
         if !m.set
           m.left_rec = true
@@ -272,14 +286,11 @@ class KPeg::FormatParser
         else
           return ans
         end
-
-        return ans
       end
     end
 
     def apply(rule)
       if m = @memoizations[rule][@pos]
-        prev = @pos
         @pos = m.pos
         if !m.set
           m.left_rec = true
@@ -307,8 +318,6 @@ class KPeg::FormatParser
         else
           return ans
         end
-
-        return ans
       end
     end
 
@@ -347,20 +356,28 @@ class KPeg::FormatParser
       RuleInfo.new(name, rendered)
     end
 
-    #
+
+  # :startdoc:
 
 
-    require 'kpeg/grammar'
+
+    ##
+    # Creates a new kpeg format parser for +str+.
 
     def initialize(str, debug=false)
       setup_parser(str, debug)
       @g = KPeg::Grammar.new
     end
 
+    ##
+    # The parsed grammar
+
     attr_reader :g
+
     alias_method :grammar, :g
 
 
+  # :stopdoc:
   def setup_foreign_grammar; end
 
   # eol = "\n"
@@ -509,7 +526,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # var = < ("-" | /[a-zA-Z][\-_a-zA-Z0-9]*/) > { text }
+  # var = < ("-" | /[a-z][\w-]*/i) > { text }
   def _var
 
     _save = self.pos
@@ -521,7 +538,7 @@ class KPeg::FormatParser
         _tmp = match_string("-")
         break if _tmp
         self.pos = _save1
-        _tmp = scan(/\A(?-mix:[a-zA-Z][\-_a-zA-Z0-9]*)/)
+        _tmp = scan(/\G(?i-mx:[a-z][\w-]*)/)
         break if _tmp
         self.pos = _save1
         break
@@ -546,13 +563,13 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # method = < /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }
+  # method = < /[a-z_]\w*/i > { text }
   def _method
 
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[a-zA-Z_][a-zA-Z0-9_]*)/)
+      _tmp = scan(/\G(?i-mx:[a-z_]\w*)/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -807,7 +824,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # num_escapes = (< /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") } | "x" < /[0-9a-fA-F]{2}/ > { [text.to_i(16)].pack("U") })
+  # num_escapes = (< /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") } | "x" < /[a-f\d]{2}/i > { [text.to_i(16)].pack("U") })
   def _num_escapes
 
     _save = self.pos
@@ -816,7 +833,7 @@ class KPeg::FormatParser
       _save1 = self.pos
       while true # sequence
         _text_start = self.pos
-        _tmp = scan(/\A(?-mix:[0-7]{1,3})/)
+        _tmp = scan(/\G(?-mix:[0-7]{1,3})/)
         if _tmp
           text = get_text(_text_start)
         end
@@ -843,7 +860,7 @@ class KPeg::FormatParser
           break
         end
         _text_start = self.pos
-        _tmp = scan(/\A(?-mix:[0-9a-fA-F]{2})/)
+        _tmp = scan(/\G(?i-mx:[a-f\d]{2})/)
         if _tmp
           text = get_text(_text_start)
         end
@@ -874,7 +891,7 @@ class KPeg::FormatParser
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[^\\"]+)/)
+      _tmp = scan(/\G(?-mix:[^\\"]+)/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -1012,7 +1029,7 @@ class KPeg::FormatParser
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[^'])/)
+      _tmp = scan(/\G(?-mix:[^'])/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -1137,7 +1154,7 @@ class KPeg::FormatParser
         _tmp = match_string("\\/")
         break if _tmp
         self.pos = _save2
-        _tmp = scan(/\A(?-mix:[^\/])/)
+        _tmp = scan(/\G(?-mix:[^\/])/)
         break if _tmp
         self.pos = _save2
         break
@@ -1151,7 +1168,7 @@ class KPeg::FormatParser
             _tmp = match_string("\\/")
             break if _tmp
             self.pos = _save3
-            _tmp = scan(/\A(?-mix:[^\/])/)
+            _tmp = scan(/\G(?-mix:[^\/])/)
             break if _tmp
             self.pos = _save3
             break
@@ -1258,13 +1275,13 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # char = < /[a-zA-Z0-9]/ > { text }
+  # char = < /[a-z\d]/i > { text }
   def _char
 
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[a-zA-Z0-9])/)
+      _tmp = scan(/\G(?i-mx:[a-z\d])/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -1328,13 +1345,13 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # range_num = < /[1-9][0-9]*/ > { text }
+  # range_num = < /[1-9]\d*/ > { text }
   def _range_num
 
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[1-9][0-9]*)/)
+      _tmp = scan(/\G(?-mix:[1-9]\d*)/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -1509,7 +1526,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # curly = "{" < (/[^{}"']+/ | string | curly)* > "}" { @g.action(text) }
+  # curly = "{" < (spaces | /[^{}"']+/ | string | curly)* > "}" { @g.action(text) }
   def _curly
 
     _save = self.pos
@@ -1524,7 +1541,10 @@ class KPeg::FormatParser
 
         _save2 = self.pos
         while true # choice
-          _tmp = scan(/\A(?-mix:[^{}"']+)/)
+          _tmp = apply(:_spaces)
+          break if _tmp
+          self.pos = _save2
+          _tmp = scan(/\G(?-mix:[^{}"']+)/)
           break if _tmp
           self.pos = _save2
           _tmp = apply(:_string)
@@ -1577,7 +1597,7 @@ class KPeg::FormatParser
 
         _save2 = self.pos
         while true # choice
-          _tmp = scan(/\A(?-mix:[^()"']+)/)
+          _tmp = scan(/\G(?-mix:[^()"']+)/)
           break if _tmp
           self.pos = _save2
           _tmp = apply(:_string)
@@ -2468,7 +2488,7 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - curly:act { @g.add_directive name, act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
+  # statement = (- var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) } | - var:v - "=" - expression:o { @g.set(v, o) } | - "%" var:name - "=" - < /[:\w]+/ > { @g.add_foreign_grammar(name, text) } | - "%%" - curly:act { @g.add_setup act } | - "%%" - var:name - curly:act { @g.add_directive name, act } | - "%%" - var:name - "=" - < (!"\n" .)+ > { @g.set_variable(name, text) })
   def _statement
 
     _save = self.pos
@@ -2614,7 +2634,7 @@ class KPeg::FormatParser
           break
         end
         _text_start = self.pos
-        _tmp = scan(/\A(?-mix:[::A-Za-z0-9_]+)/)
+        _tmp = scan(/\G(?-mix:[:\w]+)/)
         if _tmp
           text = get_text(_text_start)
         end
@@ -2906,13 +2926,13 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # ast_constant = < /[A-Z][A-Za-z0-9_]*/ > { text }
+  # ast_constant = < /[A-Z]\w*/ > { text }
   def _ast_constant
 
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[A-Z][A-Za-z0-9_]*)/)
+      _tmp = scan(/\G(?-mix:[A-Z]\w*)/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -2932,13 +2952,13 @@ class KPeg::FormatParser
     return _tmp
   end
 
-  # ast_word = < /[A-Za-z_][A-Za-z0-9_]*/ > { text }
+  # ast_word = < /[a-z_]\w*/i > { text }
   def _ast_word
 
     _save = self.pos
     while true # sequence
       _text_start = self.pos
-      _tmp = scan(/\A(?-mix:[A-Za-z_][A-Za-z0-9_]*)/)
+      _tmp = scan(/\G(?i-mx:[a-z_]\w*)/)
       if _tmp
         text = get_text(_text_start)
       end
@@ -3134,10 +3154,10 @@ class KPeg::FormatParser
   Rules[:_space] = rule_info("space", "(\" \" | \"\\t\" | eol)")
   Rules[:__hyphen_] = rule_info("-", "(space | comment)*")
   Rules[:_kleene] = rule_info("kleene", "\"*\"")
-  Rules[:_var] = rule_info("var", "< (\"-\" | /[a-zA-Z][\\-_a-zA-Z0-9]*/) > { text }")
-  Rules[:_method] = rule_info("method", "< /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }")
+  Rules[:_var] = rule_info("var", "< (\"-\" | /[a-z][\\w-]*/i) > { text }")
+  Rules[:_method] = rule_info("method", "< /[a-z_]\\w*/i > { text }")
   Rules[:_dbl_escapes] = rule_info("dbl_escapes", "(\"n\" { \"\\n\" } | \"s\" { \" \" } | \"r\" { \"\\r\" } | \"t\" { \"\\t\" } | \"v\" { \"\\v\" } | \"f\" { \"\\f\" } | \"b\" { \"\\b\" } | \"a\" { \"\\a\" } | \"e\" { \"\\e\" } | \"\\\\\" { \"\\\\\" } | \"\\\"\" { \"\\\"\" } | num_escapes | < . > { text })")
-  Rules[:_num_escapes] = rule_info("num_escapes", "(< /[0-7]{1,3}/ > { [text.to_i(8)].pack(\"U\") } | \"x\" < /[0-9a-fA-F]{2}/ > { [text.to_i(16)].pack(\"U\") })")
+  Rules[:_num_escapes] = rule_info("num_escapes", "(< /[0-7]{1,3}/ > { [text.to_i(8)].pack(\"U\") } | \"x\" < /[a-f\\d]{2}/i > { [text.to_i(16)].pack(\"U\") })")
   Rules[:_dbl_seq] = rule_info("dbl_seq", "< /[^\\\\\"]+/ > { text }")
   Rules[:_dbl_not_quote] = rule_info("dbl_not_quote", "(\"\\\\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) }")
   Rules[:_dbl_string] = rule_info("dbl_string", "\"\\\"\" dbl_not_quote:s \"\\\"\" { @g.str(s.join) }")
@@ -3149,13 +3169,13 @@ class KPeg::FormatParser
   Rules[:_not_slash] = rule_info("not_slash", "< (\"\\\\/\" | /[^\\/]/)+ > { text }")
   Rules[:_regexp_opts] = rule_info("regexp_opts", "< [a-z]* > { text }")
   Rules[:_regexp] = rule_info("regexp", "\"/\" not_slash:body \"/\" regexp_opts:opts { @g.reg body, opts }")
-  Rules[:_char] = rule_info("char", "< /[a-zA-Z0-9]/ > { text }")
+  Rules[:_char] = rule_info("char", "< /[a-z\\d]/i > { text }")
   Rules[:_char_range] = rule_info("char_range", "\"[\" char:l \"-\" char:r \"]\" { @g.range(l,r) }")
-  Rules[:_range_num] = rule_info("range_num", "< /[1-9][0-9]*/ > { text }")
+  Rules[:_range_num] = rule_info("range_num", "< /[1-9]\\d*/ > { text }")
   Rules[:_range_elem] = rule_info("range_elem", "< (range_num | kleene) > { text }")
   Rules[:_mult_range] = rule_info("mult_range", "(\"[\" - range_elem:l - \",\" - range_elem:r - \"]\" { [l == \"*\" ? nil : l.to_i, r == \"*\" ? nil : r.to_i] } | \"[\" - range_num:e - \"]\" { [e.to_i, e.to_i] })")
   Rules[:_curly_block] = rule_info("curly_block", "curly")
-  Rules[:_curly] = rule_info("curly", "\"{\" < (/[^{}\"']+/ | string | curly)* > \"}\" { @g.action(text) }")
+  Rules[:_curly] = rule_info("curly", "\"{\" < (spaces | /[^{}\"']+/ | string | curly)* > \"}\" { @g.action(text) }")
   Rules[:_nested_paren] = rule_info("nested_paren", "\"(\" (/[^()\"']+/ | string | nested_paren)* \")\"")
   Rules[:_value] = rule_info("value", "(value:v \":\" var:n { @g.t(v,n) } | value:v \"?\" { @g.maybe(v) } | value:v \"+\" { @g.many(v) } | value:v \"*\" { @g.kleene(v) } | value:v mult_range:r { @g.multiple(v, *r) } | \"&\" value:v { @g.andp(v) } | \"!\" value:v { @g.notp(v) } | \"(\" - expression:o - \")\" { o } | \"@<\" - expression:o - \">\" { @g.bounds(o) } | \"<\" - expression:o - \">\" { @g.collect(o) } | curly_block | \"~\" method:m < nested_paren? > { @g.action(\"\#{m}\#{text}\") } | \".\" { @g.dot } | \"@\" var:name < nested_paren? > !(- \"=\") { @g.invoke(name, text.empty? ? nil : text) } | \"^\" var:name < nested_paren? > { @g.foreign_invoke(\"parent\", name, text) } | \"%\" var:gram \".\" var:name < nested_paren? > { @g.foreign_invoke(gram, name, text) } | var:name < nested_paren? > !(- \"=\") { @g.ref(name, nil, text.empty? ? nil : text) } | char_range | regexp | string)")
   Rules[:_spaces] = rule_info("spaces", "(space | comment)+")
@@ -3163,13 +3183,14 @@ class KPeg::FormatParser
   Rules[:_choose_cont] = rule_info("choose_cont", "- \"|\" - values:v { v }")
   Rules[:_expression] = rule_info("expression", "(values:v choose_cont+:alts { @g.any(v, *alts) } | values)")
   Rules[:_args] = rule_info("args", "(args:a \",\" - var:n - { a + [n] } | - var:n - { [n] })")
-  Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[::A-Za-z0-9_]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - curly:act { @g.add_directive name, act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
+  Rules[:_statement] = rule_info("statement", "(- var:v \"(\" args:a \")\" - \"=\" - expression:o { @g.set(v, o, a) } | - var:v - \"=\" - expression:o { @g.set(v, o) } | - \"%\" var:name - \"=\" - < /[:\\w]+/ > { @g.add_foreign_grammar(name, text) } | - \"%%\" - curly:act { @g.add_setup act } | - \"%%\" - var:name - curly:act { @g.add_directive name, act } | - \"%%\" - var:name - \"=\" - < (!\"\\n\" .)+ > { @g.set_variable(name, text) })")
   Rules[:_statements] = rule_info("statements", "statement (- statements)?")
   Rules[:_eof] = rule_info("eof", "!.")
   Rules[:_root] = rule_info("root", "statements - eof_comment? eof")
-  Rules[:_ast_constant] = rule_info("ast_constant", "< /[A-Z][A-Za-z0-9_]*/ > { text }")
-  Rules[:_ast_word] = rule_info("ast_word", "< /[A-Za-z_][A-Za-z0-9_]*/ > { text }")
+  Rules[:_ast_constant] = rule_info("ast_constant", "< /[A-Z]\\w*/ > { text }")
+  Rules[:_ast_word] = rule_info("ast_word", "< /[a-z_]\\w*/i > { text }")
   Rules[:_ast_sp] = rule_info("ast_sp", "(\" \" | \"\\t\")*")
   Rules[:_ast_words] = rule_info("ast_words", "(ast_words:r ast_sp \",\" ast_sp ast_word:w { r + [w] } | ast_word:w { [w] })")
   Rules[:_ast_root] = rule_info("ast_root", "(ast_constant:c \"(\" ast_words:w \")\" { [c, w] } | ast_constant:c \"()\"? { [c, []] })")
+  # :startdoc:
 end

data/lib/kpeg/grammar.rb

@@ -786,7 +786,7 @@ module KPeg
     end
 
     def maybe(node, &b)
-      op = multiple Grammar.resolve(node), 0, 1, &b
+      multiple Grammar.resolve(node), 0, 1, &b
     end
 
     def many(node, &b)
@@ -834,7 +834,7 @@ module KPeg
     # Invoke a rule defined on a foreign grammar
     # == Parameters:
     # gram::
-    #   The name of the grammar that the rule will be reference from 
+    #   The name of the grammar that the rule will be reference from
     # name::
     #   The name of the rule that will be invoked
     # args::

data/lib/kpeg/grammar_renderer.rb

@@ -10,6 +10,20 @@ module KPeg
       widest = @grammar.rules.keys.sort { |a,b| a.size <=> b.size }.last
       indent = widest.size
 
+      @grammar.variables.sort.each do |name, value|
+        io.print "%% #{name} = #{value}\n"
+      end
+
+      unless @grammar.variables.empty?
+        io.print "\n"
+      end
+
+      @grammar.directives.sort_by { |name,| name }.each do |name, act|
+        io.print "%% #{name} {"
+        io.print act.action
+        io.print "}\n\n"
+      end
+
       @grammar.setup_actions.each do |act|
         io.print "%% {"
         io.print act.action

data/lib/kpeg/position.rb

@@ -1,6 +1,7 @@
 module KPeg
   module Position
     # STANDALONE START
+
     def current_column(target=pos)
       if c = string.rindex("\n", target-1)
         return target - c - 1
@@ -9,17 +10,33 @@ module KPeg
       target + 1
     end
 
-    def current_line(target=pos)
-      cur_offset = 0
-      cur_line = 0
+    if [].respond_to? :bsearch_index
+      def current_line(target=pos)
+        unless @line_offsets
+          @line_offsets = [-1]
+          total = 0
+          string.each_line do |line|
+            @line_offsets << total
+            total += line.size
+          end
+          @line_offsets << total
+        end
 
-      string.each_line do |line|
-        cur_line += 1
-        cur_offset += line.size
-        return cur_line if cur_offset >= target
+        @line_offsets.bsearch_index {|x| x >= target } || -1
       end
+    else
+      def current_line(target=pos)
+        cur_offset = 0
+        cur_line = 0
+
+        string.each_line do |line|
+          cur_line += 1
+          cur_offset += line.size
+          return cur_line if cur_offset >= target
+        end
 
-      -1
+        -1
+      end
     end
 
     def lines

data/lib/kpeg/string_escape.kpeg

@@ -7,6 +7,7 @@
 segment = < /[\w ]+/ > { text } # Don't use \s because that matchs \n
         | "\\" { "\\\\" }
         | "\n" { "\\n" }
+        | "\r" { "\\r" }
         | "\t" { "\\t" }
         | "\b" { "\\b" }
         | "\"" { "\\\"" }