kpeg 0.9.0 → 1.2.0

data/kpeg.gemspec

@@ -1,42 +1,42 @@
 # -*- encoding: utf-8 -*-
+# stub: kpeg 1.0.0.20140103162640 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "kpeg"
-  s.version = "0.8.5.20120306163408"
+  s.version = "1.0.0.20140103162640"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Eric Hodel"]
-  s.cert_chain = ["/Users/drbrain/.gem/gem-public_cert.pem"]
-  s.date = "2012-03-07"
+  s.authors = ["Evan Phoenix"]
+  s.date = "2014-01-04"
   s.description = "KPeg is a simple PEG library for Ruby. It provides an API as well as native\ngrammar to build the grammar.\n\nKPeg strives to provide a simple, powerful API without being too exotic.\n\nKPeg supports direct left recursion of rules via the\n{OMeta memoization}[http://www.vpri.org/pdf/tr2008003_experimenting.pdf] trick."
-  s.email = ["drbrain@segment7.net"]
+  s.email = ["evan@fallingsnow.net"]
   s.executables = ["kpeg"]
-  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.rdoc"]
-  s.files = [".autotest", "Gemfile", "Gemfile.lock", "History.txt", "LICENSE", "Manifest.txt", "README.rdoc", "Rakefile", "bin/kpeg", "examples/calculator/calculator.kpeg", "examples/calculator/calculator.rb", "examples/foreign_reference/literals.kpeg", "examples/foreign_reference/matcher.kpeg", "examples/foreign_reference/matcher.rb", "examples/lua_string/driver.rb", "examples/lua_string/lua_string.kpeg", "examples/lua_string/lua_string.kpeg.rb", "examples/phone_number/README.md", "examples/phone_number/phone_number.kpeg", "examples/phone_number/phone_number.rb", "examples/upper/README.md", "examples/upper/upper.kpeg", "examples/upper/upper.rb", "kpeg.gemspec", "lib/kpeg.rb", "lib/kpeg/code_generator.rb", "lib/kpeg/compiled_parser.rb", "lib/kpeg/format.kpeg", "lib/kpeg/format_parser.rb", "lib/kpeg/grammar.rb", "lib/kpeg/grammar_renderer.rb", "lib/kpeg/match.rb", "lib/kpeg/parser.rb", "lib/kpeg/position.rb", "lib/kpeg/string_escape.kpeg", "lib/kpeg/string_escape.rb", "lib/kpeg/version.rb", "test/inputs/comments.kpeg", "test/test_file_parser_roundtrip.rb", "test/test_gen_calc.rb", "test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_grammar_renderer.rb", "test/test_left_recursion.rb", "vim/syntax_kpeg/ftdetect/kpeg.vim", "vim/syntax_kpeg/syntax/kpeg.vim", ".gemtest"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.rdoc", "examples/phone_number/README.md", "examples/upper/README.md"]
+  s.files = [".autotest", ".travis.yml", "History.txt", "LICENSE", "Manifest.txt", "README.rdoc", "Rakefile", "bin/kpeg", "examples/calculator/calculator.kpeg", "examples/calculator/calculator.rb", "examples/foreign_reference/literals.kpeg", "examples/foreign_reference/matcher.kpeg", "examples/foreign_reference/matcher.rb", "examples/lua_string/driver.rb", "examples/lua_string/lua_string.kpeg", "examples/lua_string/lua_string.kpeg.rb", "examples/phone_number/README.md", "examples/phone_number/phone_number.kpeg", "examples/phone_number/phone_number.rb", "examples/upper/README.md", "examples/upper/upper.kpeg", "examples/upper/upper.rb", "kpeg.gemspec", "lib/hoe/kpeg.rb", "lib/kpeg.rb", "lib/kpeg/code_generator.rb", "lib/kpeg/compiled_parser.rb", "lib/kpeg/format_parser.kpeg", "lib/kpeg/format_parser.rb", "lib/kpeg/grammar.rb", "lib/kpeg/grammar_renderer.rb", "lib/kpeg/match.rb", "lib/kpeg/parser.rb", "lib/kpeg/position.rb", "lib/kpeg/string_escape.kpeg", "lib/kpeg/string_escape.rb", "test/inputs/comments.kpeg", "test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_format_parser_round_trip.rb", "test/test_kpeg_grammar.rb", "test/test_kpeg_grammar_renderer.rb", "vim/syntax_kpeg/ftdetect/kpeg.vim", "vim/syntax_kpeg/syntax/kpeg.vim", "test/test_kpeg_string_escape.rb", ".gemtest"]
   s.homepage = "https://github.com/evanphx/kpeg"
+  s.licenses = ["MIT"]
   s.rdoc_options = ["--main", "README.rdoc"]
   s.require_paths = ["lib"]
   s.rubyforge_project = "kpeg"
-  s.rubygems_version = "1.8.12"
-  s.signing_key = "/Users/drbrain/.gem/gem-private_key.pem"
+  s.rubygems_version = "2.1.10"
   s.summary = "KPeg is a simple PEG library for Ruby"
-  s.test_files = ["test/test_file_parser_roundtrip.rb", "test/test_gen_calc.rb", "test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_grammar_renderer.rb", "test/test_left_recursion.rb"]
+  s.test_files = ["test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_format_parser_round_trip.rb", "test/test_kpeg_grammar.rb", "test/test_kpeg_grammar_renderer.rb", "test/test_kpeg_string_escape.rb"]
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<minitest>, ["~> 2.11"])
-      s.add_development_dependency(%q<rdoc>, ["~> 3.10"])
-      s.add_development_dependency(%q<hoe>, ["~> 2.15"])
+      s.add_development_dependency(%q<minitest>, ["~> 5.2"])
+      s.add_development_dependency(%q<rdoc>, ["~> 4.0"])
+      s.add_development_dependency(%q<hoe>, ["~> 3.7"])
     else
-      s.add_dependency(%q<minitest>, ["~> 2.11"])
-      s.add_dependency(%q<rdoc>, ["~> 3.10"])
-      s.add_dependency(%q<hoe>, ["~> 2.15"])
+      s.add_dependency(%q<minitest>, ["~> 5.2"])
+      s.add_dependency(%q<rdoc>, ["~> 4.0"])
+      s.add_dependency(%q<hoe>, ["~> 3.7"])
     end
   else
-    s.add_dependency(%q<minitest>, ["~> 2.11"])
-    s.add_dependency(%q<rdoc>, ["~> 3.10"])
-    s.add_dependency(%q<hoe>, ["~> 2.15"])
+    s.add_dependency(%q<minitest>, ["~> 5.2"])
+    s.add_dependency(%q<rdoc>, ["~> 4.0"])
+    s.add_dependency(%q<hoe>, ["~> 3.7"])
   end
 end

data/lib/hoe/kpeg.rb

@@ -10,7 +10,7 @@
 # license:
 #
 # Copyright (c) Ryan Davis, seattle.rb
-# 
+#
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
@@ -18,10 +18,10 @@
 # distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
-# 
+#
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
-# 
+#
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
@@ -55,9 +55,10 @@ module Hoe::Kpeg
   def initialize_kpeg
     self.kpeg_tasks = [:multi, :test, :check_manifest]
 
-    # -v = verbose
+    # -f = overwrite existing file
     # -s = parser does not require runtime
-    self.kpeg_flags ||= "-s -v"
+    # -v = verbose
+    self.kpeg_flags ||= "-s -v -f"
 
     dependency 'kpeg', '~> 0.9', :development
   end

data/lib/kpeg/code_generator.rb

@@ -83,18 +83,21 @@ module KPeg
 
       if output_node
         code << "  end\n"
+        code << "  module #{root}Construction\n"
         methods.each do |short, name, attrs|
-          code << "  def #{short}(#{attrs.join(', ')})\n"
-          code << "    #{root}::#{name}.new(#{attrs.join(', ')})\n"
-          code << "  end\n"
+          code << "    def #{short}(#{attrs.join(', ')})\n"
+          code << "      #{root}::#{name}.new(#{attrs.join(', ')})\n"
+          code << "    end\n"
         end
+        code << "  end\n"
+        code << "  include #{root}Construction\n"
       end
     end
-    
+
     def indentify(code, indent)
       "#{"  " * indent}#{code}"
     end
-    
+
     # Default indent is 4 spaces (indent=2)
     def output_op(code, op, indent=2)
       case op
@@ -109,7 +112,7 @@ module KPeg
           # Let default ruby string handling figure it out
           lang = ""
         end
-        code << indentify("_tmp = scan(/\\A#{op.regexp}/#{lang})\n", indent)
+        code << indentify("_tmp = scan(/\\G#{op.regexp}/#{lang})\n", indent)
       when CharRange
         ss = save()
         if op.start.bytesize == 1 and op.fin.bytesize == 1
@@ -124,7 +127,7 @@ module KPeg
             left  = op.start[0]
             right = op.fin[0]
           end
-          
+
           code << indentify("  unless _tmp >= #{left} and _tmp <= #{right}\n", indent)
           code << indentify("    self.pos = #{ss}\n", indent)
           code << indentify("    _tmp = nil\n", indent)
@@ -140,7 +143,7 @@ module KPeg
         code << indentify("while true # choice\n", indent)
         op.ops.each_with_index do |n,idx|
           output_op code, n, (indent+1)
-          
+
           code << indentify("  break if _tmp\n", indent)
           code << indentify("  self.pos = #{ss}\n", indent)
           if idx == op.ops.size - 1
@@ -309,15 +312,23 @@ module KPeg
       else
         raise "Unknown op - #{op.class}"
       end
-
     end
 
-    def standalone_region(path)
-      cp = File.read(path)
-      start = cp.index("# STANDALONE START")
-      fin = cp.index("# STANDALONE END")
+    def standalone_region(path, marker = "STANDALONE")
+      expanded_path = File.expand_path("../#{path}", __FILE__)
+      cp = File.read(expanded_path)
+
+      start_marker = "# #{marker} START"
+      end_marker   = /^\s*# #{Regexp.escape marker} END/
+
+      start = cp.index(start_marker) + start_marker.length + 1 # \n
+      fin   = cp.index(end_marker)
+
+      unless start and fin
+        abort("#{marker} boundaries in #{path} missing " \
+              "for standalone generation")
+      end
 
-      return nil unless start and fin
       cp[start..fin]
     end
 
@@ -326,37 +337,29 @@ module KPeg
 
       code = []
 
-      if header = @grammar.directives['header']
-        code << header.action.strip
-        code << "\n"
-      end
+      output_header(code)
+      output_grammar(code)
+      output_footer(code)
 
-      if @standalone
-        code << "class #{@name}\n"
-
-        unless cp = standalone_region(
-                    File.expand_path("../compiled_parser.rb", __FILE__))
+      @output = code.join
+    end
 
-          puts "Standalone failure. Check compiler_parser.rb for proper boundary comments"
-          exit 1
-        end
+    ##
+    # Output of class end and footer
 
-        unless pp = standalone_region(
-                    File.expand_path("../position.rb", __FILE__))
-          puts "Standalone failure. Check position.rb for proper boundary comments"
-        end
+    def output_footer(code)
+      code << "end\n"
 
-        cp.gsub!(/include Position/, pp)
-        code << cp << "\n"
-      else
-        code << "require 'kpeg/compiled_parser'\n\n"
-        code << "class #{@name} < KPeg::CompiledParser\n"
+      if footer = @grammar.directives['footer']
+        code << footer.action
       end
+    end
 
-      @grammar.setup_actions.each do |act|
-        code << "\n#{act.action}\n\n"
-      end
+    ##
+    # Output of grammar and rules
 
+    def output_grammar(code)
+      code << "  # :stopdoc:\n"
       handle_ast(code)
 
       fg = @grammar.foreign_grammars
@@ -418,24 +421,57 @@ module KPeg
 
       code << "\n  Rules = {}\n"
       @grammar.rule_order.each do |name|
-        rule = @grammar.rules[name]
-
         rend = GrammarRenderer.escape renderings[name], true
         code << "  Rules[:#{method_name name}] = rule_info(\"#{name}\", \"#{rend}\")\n"
       end
 
-      code << "end\n"
+      code << "  # :startdoc:\n"
+    end
 
-      if footer = @grammar.directives['footer']
-        code << footer.action
+    ##
+    # Output up to the user-defined setup actions
+
+    def output_header(code)
+      if header = @grammar.directives['header']
+        code << header.action.strip
+        code << "\n"
       end
 
-      @output = code.join
+      pre_class = @grammar.directives['pre-class']
+
+      if @standalone
+        if pre_class
+          code << pre_class.action.strip
+          code << "\n"
+        end
+        code << "class #{@name}\n"
+
+        cp  = standalone_region("compiled_parser.rb")
+        cpi = standalone_region("compiled_parser.rb", "INITIALIZE")
+        pp  = standalone_region("position.rb")
+
+        cp.gsub!(/^\s*include Position/, pp)
+        code << "  # :stopdoc:\n"
+        code << cpi << "\n" unless @grammar.variables['custom_initialize']
+        code << cp  << "\n"
+        code << "  # :startdoc:\n"
+      else
+        code << "require 'kpeg/compiled_parser'\n\n"
+        if pre_class
+          code << pre_class.action.strip
+          code << "\n"
+        end
+        code << "class #{@name} < KPeg::CompiledParser\n"
+      end
+
+      @grammar.setup_actions.each do |act|
+        code << "\n#{act.action}\n\n"
+      end
     end
 
     def make(str)
       m = Module.new
-      m.module_eval output
+      m.module_eval output, "(kpeg parser #{@name})"
 
       cls = m.const_get(@name)
       cls.new(str)

data/lib/kpeg/compiled_parser.rb

@@ -10,27 +10,33 @@ module KPeg
 
     # Leave these markers in! They allow us to generate standalone
     # code automatically!
-    #
+
+    # INITIALIZE START
+
+    # This is distinct from setup_parser so that a standalone parser
+    # can redefine #initialize and still have access to the proper
+    # parser setup code.
+    def initialize(str, debug=false)
+      setup_parser(str, debug)
+    end
+
+    # INITIALIZE END
+
     # STANDALONE START
+
+    # Prepares for parsing +str+.  If you define a custom initialize you must
+    # call this method before #parse
     def setup_parser(str, debug=false)
-      @string = str
-      @pos = 0
+      set_string str, 0
       @memoizations = Hash.new { |h,k| h[k] = {} }
       @result = nil
       @failed_rule = nil
       @failing_rule_offset = -1
+      @line_offsets = nil
 
       setup_foreign_grammar
     end
 
-    # This is distinct from setup_parser so that a standalone parser
-    # can redefine #initialize and still have access to the proper
-    # parser setup code.
-    #
-    def initialize(str, debug=false)
-      setup_parser(str, debug)
-    end
-
     attr_reader :string
     attr_reader :failing_rule_offset
     attr_accessor :result, :pos
@@ -41,6 +47,13 @@ module KPeg
       @string[start..@pos-1]
     end
 
+    # Sets the string and current parsing position for the parser.
+    def set_string string, pos
+      @string = string
+      @string_size = string ? string.size : 0
+      @pos = pos
+    end
+
     def show_pos
       width = 10
       if @pos < width
@@ -138,28 +151,27 @@ module KPeg
     end
 
     def scan(reg)
-      if m = reg.match(@string[@pos..-1])
-        width = m.end(0)
-        @pos += width
+      if m = reg.match(@string, @pos)
+        @pos = m.end(0)
         return true
       end
 
       return nil
     end
 
-    if "".respond_to? :getbyte
+    if "".respond_to? :ord
       def get_byte
-        if @pos >= @string.size
+        if @pos >= @string_size
           return nil
         end
 
-        s = @string.getbyte @pos
+        s = @string[@pos].ord
         @pos += 1
         s
       end
     else
       def get_byte
-        if @pos >= @string.size
+        if @pos >= @string_size
           return nil
         end
 
@@ -208,8 +220,7 @@ module KPeg
       old_pos = @pos
       old_string = @string
 
-      @pos = other.pos
-      @string = other.string
+      set_string other.string, other.pos
 
       begin
         if val = __send__(rule, *args)
@@ -220,15 +231,13 @@ module KPeg
         end
         val
       ensure
-        @pos = old_pos
-        @string = old_string
+        set_string old_string, old_pos
       end
     end
 
     def apply_with_args(rule, *args)
       memo_key = [rule, args]
       if m = @memoizations[memo_key][@pos]
-        prev = @pos
         @pos = m.pos
         if !m.set
           m.left_rec = true
@@ -256,14 +265,11 @@ module KPeg
         else
           return ans
         end
-
-        return ans
       end
     end
 
     def apply(rule)
       if m = @memoizations[rule][@pos]
-        prev = @pos
         @pos = m.pos
         if !m.set
           m.left_rec = true
@@ -291,8 +297,6 @@ module KPeg
         else
           return ans
         end
-
-        return ans
       end
     end
 

data/lib/kpeg/format_parser.kpeg

@@ -1,14 +1,25 @@
 %% name = KPeg::FormatParser
+%% custom_initialize = true
+
+%% pre-class {
+require 'kpeg/grammar'
+}
 
 %% {
-    require 'kpeg/grammar'
+
+    ##
+    # Creates a new kpeg format parser for +str+.
 
     def initialize(str, debug=false)
       setup_parser(str, debug)
       @g = KPeg::Grammar.new
     end
 
+    ##
+    # The parsed grammar
+
     attr_reader :g
+
     alias_method :grammar, :g
 }
 
@@ -22,8 +33,8 @@
           kleene = "*"
 
                    # Allow - by itself, but not at the beginning
-             var = < "-" | /[a-zA-Z][\-_a-zA-Z0-9]*/ > { text }
-          method = < /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }
+             var = < "-" | /[a-z][\w-]*/i > { text }
+          method = < /[a-z_]\w*/i > { text }
 
      dbl_escapes = "n" { "\n" }
                  | "s" { " " }
@@ -39,7 +50,8 @@
                  | num_escapes
                  | < . > { text }
      num_escapes = < /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") }
-                 | "x" < /[0-9a-fA-F]{2}/ > { [text.to_i(16)].pack("U") }
+                 | "x" < /[a-f\d]{2}/i > { [text.to_i(16)].pack("U") }
+                 # TODO use /\h{2}/ after 1.8 support is dropped
          dbl_seq = < /[^\\"]+/ > { text }
    dbl_not_quote = ("\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) }
       dbl_string = "\"" dbl_not_quote:s "\"" { @g.str(s.join) }
@@ -55,17 +67,17 @@ sgl_escape_quote = "\\'" { "'" }
           regexp = "/" not_slash:body "/" regexp_opts:opts
                    { @g.reg body, opts }
 
-            char = < /[a-zA-Z0-9]/ > { text }
+            char = < /[a-z\d]/i > { text }
       char_range = "[" char:l "-" char:r "]" { @g.range(l,r) }
 
-       range_num = < /[1-9][0-9]*/ > { text }
+       range_num = < /[1-9]\d*/ > { text }
       range_elem = < range_num|kleene > { text }
       mult_range = "[" - range_elem:l - "," - range_elem:r - "]"
                    { [l == "*" ? nil : l.to_i, r == "*" ? nil : r.to_i] }
                  | "[" - range_num:e - "]" { [e.to_i, e.to_i] }
 
      curly_block = curly
-           curly = "{" < (/[^{}"']+/ | string | curly)* > "}" { @g.action(text) }
+           curly = "{" < (spaces | /[^{}"']+/ | string | curly)* > "}" { @g.action(text) }
     nested_paren = "(" (/[^()"']+/ | string | nested_paren)* ")"
 
            value = value:v ":" var:n { @g.t(v,n) }
@@ -105,7 +117,7 @@ sgl_escape_quote = "\\'" { "'" }
                  | - var:n - { [n] }
        statement = - var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) }
                  | - var:v - "=" - expression:o { @g.set(v, o) }
-                 | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ >
+                 | - "%" var:name - "=" - < /[:\w]+/ >
                    { @g.add_foreign_grammar(name, text) }
                  | - "%%" - curly:act { @g.add_setup act }
                  | - "%%" - var:name - curly:act { @g.add_directive name, act }
@@ -117,8 +129,8 @@ sgl_escape_quote = "\\'" { "'" }
 
 # These are a seperate set of rules used to parse an ast declaration
 
-    ast_constant = < /[A-Z][A-Za-z0-9_]*/ > { text }
-        ast_word = < /[A-Za-z_][A-Za-z0-9_]*/ > { text }
+    ast_constant = < /[A-Z]\w*/ > { text }
+        ast_word = < /[a-z_]\w*/i > { text }
 
           ast_sp = (" " | "\t")*