kpeg 0.9.0 → 0.10.0

data/History.txt

@@ -1,4 +1,18 @@
-=== 1.0 / 2012-04-06
+=== 0.10 / 2012-04-16
+
+* Minor enhancements
+  * In standalone parsers generation of a default initialize method may be
+    disabled with the custom_initialize variable:
+
+      %% custom_initialize = true
+  * Added a pre-class directive for adding class comments
+  * Generated code is now surrounded by startdoc/stopdoc.
+
+* Bug fixes
+  * Hoe plugin now overwrites generated files
+  * Directives and variables now round-trip through KPeg::GrammarRenderer
+
+=== 0.9 / 2012-04-06
 
 * Minor enhancements
   * Added arbitrary directives to the kpeg grammar

data/Manifest.txt

@@ -1,5 +1,4 @@
 .autotest
-Gemfile
 History.txt
 LICENSE
 Manifest.txt
@@ -35,13 +34,12 @@ lib/kpeg/position.rb
 lib/kpeg/string_escape.kpeg
 lib/kpeg/string_escape.rb
 test/inputs/comments.kpeg
-test/test_file_parser_roundtrip.rb
-test/test_gen_calc.rb
 test/test_kpeg.rb
 test/test_kpeg_code_generator.rb
 test/test_kpeg_compiled_parser.rb
 test/test_kpeg_format.rb
+test/test_kpeg_format_parser_round_trip.rb
+test/test_kpeg_grammar.rb
 test/test_kpeg_grammar_renderer.rb
-test/test_left_recursion.rb
 vim/syntax_kpeg/ftdetect/kpeg.vim
 vim/syntax_kpeg/syntax/kpeg.vim

data/README.rdoc

@@ -157,6 +157,38 @@ Kpeg allows comments to be added to the grammar file by using the # symbol
 
   # This is a comment in my grammar
 
+=== Variables
+
+A variable looks like this:
+
+  %% name = value
+
+Kpeg allows the following variables that control the output parser:
+
+name::
+  The class name of the generated parser.
+custom_initialize::
+  When built as a standalone parser a default initialize method will not be
+  included.
+
+=== Directives
+
+A directive looks like this:
+
+  %% header {
+    ...
+  }
+
+Kpeg allows the following directives:
+
+header::
+  Placed before any generated code
+pre-class::
+  Placed before the class definition to provide a class comment
+footer::
+  Placed after the end of the class (for requiring files dependent upon the
+  parser's namespace
+
 == Generating and running your parser
 
 Before you can generate your parser you will need to define a root rule. This

data/Rakefile

@@ -3,7 +3,6 @@
 require 'rubygems'
 require 'hoe'
 
-Hoe.plugin :bundler
 Hoe.plugin :gemspec
 Hoe.plugin :git
 Hoe.plugin :minitest
@@ -29,10 +28,18 @@ rule ".rb" => ".kpeg" do |t|
   ruby "-Ilib bin/kpeg -s -o #{t.name} -f #{t.source}"
 end
 
-desc "build the parser"
-task :parser => %w[
+PARSER_FILES = %w[
   lib/kpeg/string_escape.rb
   lib/kpeg/format_parser.rb
 ]
 
+PARSER_FILES.map do |parser_file|
+  file parser_file => 'lib/kpeg/compiled_parser.rb'
+  file parser_file => 'lib/kpeg/code_generator.rb'
+  file parser_file => 'lib/kpeg/position.rb'
+end
+
+desc "build the parser"
+task :parser => PARSER_FILES
+
 # vim: syntax=ruby

data/lib/hoe/kpeg.rb

@@ -10,7 +10,7 @@
 # license:
 #
 # Copyright (c) Ryan Davis, seattle.rb
-# 
+#
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
 # "Software"), to deal in the Software without restriction, including
@@ -18,10 +18,10 @@
 # distribute, sublicense, and/or sell copies of the Software, and to
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
-# 
+#
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
-# 
+#
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
@@ -55,9 +55,10 @@ module Hoe::Kpeg
   def initialize_kpeg
     self.kpeg_tasks = [:multi, :test, :check_manifest]
 
-    # -v = verbose
+    # -f = overwrite existing file
     # -s = parser does not require runtime
-    self.kpeg_flags ||= "-s -v"
+    # -v = verbose
+    self.kpeg_flags ||= "-s -v -f"
 
     dependency 'kpeg', '~> 0.9', :development
   end

data/lib/kpeg.rb

@@ -1,6 +1,6 @@
 module KPeg
 
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 
   def self.grammar
     g = Grammar.new

data/lib/kpeg/code_generator.rb

@@ -90,11 +90,11 @@ module KPeg
         end
       end
     end
-    
+
     def indentify(code, indent)
       "#{"  " * indent}#{code}"
     end
-    
+
     # Default indent is 4 spaces (indent=2)
     def output_op(code, op, indent=2)
       case op
@@ -309,15 +309,23 @@ module KPeg
       else
         raise "Unknown op - #{op.class}"
       end
-
     end
 
-    def standalone_region(path)
-      cp = File.read(path)
-      start = cp.index("# STANDALONE START")
-      fin = cp.index("# STANDALONE END")
+    def standalone_region(path, marker = "STANDALONE")
+      expanded_path = File.expand_path("../#{path}", __FILE__)
+      cp = File.read(expanded_path)
+
+      start_marker = "# #{marker} START"
+      end_marker   = /^\s*# #{Regexp.escape marker} END/
+
+      start = cp.index(start_marker) + start_marker.length + 1 # \n
+      fin   = cp.index(end_marker)
+
+      unless start and fin
+        abort("#{marker} boundaries in #{path} missing " \
+              "for standalone generation")
+      end
 
-      return nil unless start and fin
       cp[start..fin]
     end
 
@@ -326,37 +334,29 @@ module KPeg
 
       code = []
 
-      if header = @grammar.directives['header']
-        code << header.action.strip
-        code << "\n"
-      end
-
-      if @standalone
-        code << "class #{@name}\n"
+      output_header(code)
+      output_grammar(code)
+      output_footer(code)
 
-        unless cp = standalone_region(
-                    File.expand_path("../compiled_parser.rb", __FILE__))
+      @output = code.join
+    end
 
-          puts "Standalone failure. Check compiler_parser.rb for proper boundary comments"
-          exit 1
-        end
+    ##
+    # Output of class end and footer
 
-        unless pp = standalone_region(
-                    File.expand_path("../position.rb", __FILE__))
-          puts "Standalone failure. Check position.rb for proper boundary comments"
-        end
+    def output_footer(code)
+      code << "end\n"
 
-        cp.gsub!(/include Position/, pp)
-        code << cp << "\n"
-      else
-        code << "require 'kpeg/compiled_parser'\n\n"
-        code << "class #{@name} < KPeg::CompiledParser\n"
+      if footer = @grammar.directives['footer']
+        code << footer.action
       end
+    end
 
-      @grammar.setup_actions.each do |act|
-        code << "\n#{act.action}\n\n"
-      end
+    ##
+    # Output of grammar and rules
 
+    def output_grammar(code)
+      code << "  # :stopdoc:\n"
       handle_ast(code)
 
       fg = @grammar.foreign_grammars
@@ -418,24 +418,57 @@ module KPeg
 
       code << "\n  Rules = {}\n"
       @grammar.rule_order.each do |name|
-        rule = @grammar.rules[name]
-
         rend = GrammarRenderer.escape renderings[name], true
         code << "  Rules[:#{method_name name}] = rule_info(\"#{name}\", \"#{rend}\")\n"
       end
 
-      code << "end\n"
+      code << "  # :startdoc:\n"
+    end
 
-      if footer = @grammar.directives['footer']
-        code << footer.action
+    ##
+    # Output up to the user-defined setup actions
+
+    def output_header(code)
+      if header = @grammar.directives['header']
+        code << header.action.strip
+        code << "\n"
       end
 
-      @output = code.join
+      pre_class = @grammar.directives['pre-class']
+
+      if @standalone
+        if pre_class
+          code << pre_class.action.strip
+          code << "\n"
+        end
+        code << "class #{@name}\n"
+
+        cp  = standalone_region("compiled_parser.rb")
+        cpi = standalone_region("compiled_parser.rb", "INITIALIZE")
+        pp  = standalone_region("position.rb")
+
+        cp.gsub!(/include Position/, pp)
+        code << "  # :stopdoc:\n"
+        code << cpi << "\n" unless @grammar.variables['custom_initialize']
+        code << cp  << "\n"
+        code << "  # :startdoc:\n"
+      else
+        code << "require 'kpeg/compiled_parser'\n\n"
+        if pre_class
+          code << pre_class.action.strip
+          code << "\n"
+        end
+        code << "class #{@name} < KPeg::CompiledParser\n"
+      end
+
+      @grammar.setup_actions.each do |act|
+        code << "\n#{act.action}\n\n"
+      end
     end
 
     def make(str)
       m = Module.new
-      m.module_eval output
+      m.module_eval output, "(kpeg parser #{@name})"
 
       cls = m.const_get(@name)
       cls.new(str)

data/lib/kpeg/compiled_parser.rb

@@ -10,8 +10,22 @@ module KPeg
 
     # Leave these markers in! They allow us to generate standalone
     # code automatically!
-    #
+
+    # INITIALIZE START
+
+    # This is distinct from setup_parser so that a standalone parser
+    # can redefine #initialize and still have access to the proper
+    # parser setup code.
+    def initialize(str, debug=false)
+      setup_parser(str, debug)
+    end
+
+    # INITIALIZE END
+
     # STANDALONE START
+
+    # Prepares for parsing +str+.  If you define a custom initialize you must
+    # call this method before #parse
     def setup_parser(str, debug=false)
       @string = str
       @pos = 0
@@ -23,14 +37,6 @@ module KPeg
       setup_foreign_grammar
     end
 
-    # This is distinct from setup_parser so that a standalone parser
-    # can redefine #initialize and still have access to the proper
-    # parser setup code.
-    #
-    def initialize(str, debug=false)
-      setup_parser(str, debug)
-    end
-
     attr_reader :string
     attr_reader :failing_rule_offset
     attr_accessor :result, :pos
@@ -228,7 +234,6 @@ module KPeg
     def apply_with_args(rule, *args)
       memo_key = [rule, args]
       if m = @memoizations[memo_key][@pos]
-        prev = @pos
         @pos = m.pos
         if !m.set
           m.left_rec = true
@@ -263,7 +268,6 @@ module KPeg
 
     def apply(rule)
       if m = @memoizations[rule][@pos]
-        prev = @pos
         @pos = m.pos
         if !m.set
           m.left_rec = true

data/lib/kpeg/format_parser.kpeg

@@ -1,14 +1,25 @@
 %% name = KPeg::FormatParser
+%% custom_initialize = true
+
+%% pre-class {
+require 'kpeg/grammar'
+}
 
 %% {
-    require 'kpeg/grammar'
+
+    ##
+    # Creates a new kpeg format parser for +str+.
 
     def initialize(str, debug=false)
       setup_parser(str, debug)
       @g = KPeg::Grammar.new
     end
 
+    ##
+    # The parsed grammar
+
     attr_reader :g
+
     alias_method :grammar, :g
 }
 
@@ -22,8 +33,8 @@
           kleene = "*"
 
                    # Allow - by itself, but not at the beginning
-             var = < "-" | /[a-zA-Z][\-_a-zA-Z0-9]*/ > { text }
-          method = < /[a-zA-Z_][a-zA-Z0-9_]*/ > { text }
+             var = < "-" | /[a-z][\w-]*/i > { text }
+          method = < /[a-z_]\w*/i > { text }
 
      dbl_escapes = "n" { "\n" }
                  | "s" { " " }
@@ -39,7 +50,8 @@
                  | num_escapes
                  | < . > { text }
      num_escapes = < /[0-7]{1,3}/ > { [text.to_i(8)].pack("U") }
-                 | "x" < /[0-9a-fA-F]{2}/ > { [text.to_i(16)].pack("U") }
+                 | "x" < /[a-f\d]{2}/i > { [text.to_i(16)].pack("U") }
+                 # TODO use /\h{2}/ after 1.8 support is dropped
          dbl_seq = < /[^\\"]+/ > { text }
    dbl_not_quote = ("\\" dbl_escapes:s | dbl_seq:s)*:ary { Array(ary) }
       dbl_string = "\"" dbl_not_quote:s "\"" { @g.str(s.join) }
@@ -55,10 +67,10 @@ sgl_escape_quote = "\\'" { "'" }
           regexp = "/" not_slash:body "/" regexp_opts:opts
                    { @g.reg body, opts }
 
-            char = < /[a-zA-Z0-9]/ > { text }
+            char = < /[a-z\d]/i > { text }
       char_range = "[" char:l "-" char:r "]" { @g.range(l,r) }
 
-       range_num = < /[1-9][0-9]*/ > { text }
+       range_num = < /[1-9]\d*/ > { text }
       range_elem = < range_num|kleene > { text }
       mult_range = "[" - range_elem:l - "," - range_elem:r - "]"
                    { [l == "*" ? nil : l.to_i, r == "*" ? nil : r.to_i] }
@@ -105,7 +117,7 @@ sgl_escape_quote = "\\'" { "'" }
                  | - var:n - { [n] }
        statement = - var:v "(" args:a ")" - "=" - expression:o { @g.set(v, o, a) }
                  | - var:v - "=" - expression:o { @g.set(v, o) }
-                 | - "%" var:name - "=" - < /[::A-Za-z0-9_]+/ >
+                 | - "%" var:name - "=" - < /[:\w]+/ >
                    { @g.add_foreign_grammar(name, text) }
                  | - "%%" - curly:act { @g.add_setup act }
                  | - "%%" - var:name - curly:act { @g.add_directive name, act }
@@ -117,8 +129,8 @@ sgl_escape_quote = "\\'" { "'" }
 
 # These are a seperate set of rules used to parse an ast declaration
 
-    ast_constant = < /[A-Z][A-Za-z0-9_]*/ > { text }
-        ast_word = < /[A-Za-z_][A-Za-z0-9_]*/ > { text }
+    ast_constant = < /[A-Z]\w*/ > { text }
+        ast_word = < /[a-z_]\w*/i > { text }
 
           ast_sp = (" " | "\t")*