kpeg 0.7

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in callisto.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2011, Evan Phoenix
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the <organization> nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/README.md

@@ -0,0 +1,8 @@
+KPeg
+====
+
+KPeg is a simple PEG library for Ruby. It provides an API as well as native grammar to build the grammar.
+
+KPeg strives to provide a simple, powerful API without being too exotic.
+
+KPeg supports direct left recursion of rules via the [OMeta memoization](http://www.vpri.org/pdf/tr2008003_experimenting.pdf) trick.

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rake/testtask'
+
+$:.unshift "lib"
+
+task :default => :test
+
+desc "Run tests"
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end
+
+task :grammar do
+  require 'kpeg'
+  require 'kpeg/format'
+  require 'kpeg/grammar_renderer'
+
+  gr = KPeg::GrammarRenderer.new(KPeg::FORMAT)
+  gr.render(STDOUT)
+end
+
+task :parser do
+  sh "ruby -Ilib bin/kpeg -o lib/kpeg/format_parser.rb -s -f lib/kpeg/format.kpeg"
+end

data/bin/kpeg

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+require 'kpeg'
+require 'kpeg/code_generator'
+require 'kpeg/format_parser'
+require 'kpeg/grammar_renderer'
+
+require 'optparse'
+
+options = {}
+OptionParser.new do |o|
+  o.banner = "Usage: kpeg [options]"
+
+  o.on("-t", "--test", "Syntax check the file only") do |v|
+    options[:test] = v
+  end
+
+  o.on("--reformat", "Reformat your grammar and write it back out") do
+    options[:reformat] = true
+  end
+
+  o.on("-o", "--output FILE", "Where the output should go") do |v|
+    options[:output] = v
+  end
+
+  o.on("-n", "--name NAME", "Class name to use for the parser") do |v|
+    options[:name] = v
+  end
+
+  o.on("-f", "--force", "Overwrite the output if it exists") do |v|
+    options[:force] = v
+  end
+
+  o.on("-s", "--stand-alone", "Write the parser to run standalone") do |v|
+    options[:standalone] = v
+  end
+
+  o.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+    options[:verbose] = v
+  end
+
+  o.on("-d", "--debug", "Debug parsing the file") do |v|
+    options[:debug] = v
+  end
+end.parse!
+
+file = ARGV.shift
+
+unless File.exists?(file)
+  puts "File '#{file}' does not exist"
+  exit 1
+end
+
+parser = KPeg::FormatParser.new File.read(file), true
+
+unless m = parser.parse
+  puts "Syntax error in grammar #{file}"
+  parser.show_error
+  exit 1
+end
+
+grammar = parser.grammar
+
+if options[:reformat]
+  if !options[:output]
+    puts "Please specify -o for where to write the new grammar"
+    exit 1
+  end
+
+  output = options[:output]
+  if File.exists?(output) and !options[:force]
+    puts "Output '#{output}' already exists, not overwriting (use -f)"
+    exit 1
+  end
+
+  rend = KPeg::GrammarRenderer.new(parser.grammar)
+
+  File.open output, "w" do |f|
+    rend.render(f)
+  end
+
+  puts "Wrote reformatted output to #{output}"
+
+  exit 0
+end
+
+if !options[:test] and !options[:name]
+  unless name = grammar.variables["name"]
+    puts "Please specify -n"
+    exit 1
+  end
+else
+  name = options[:name]
+end
+
+
+if options[:output]
+  new_path = options[:output]
+else
+  new_path = "#{file}.rb"
+end
+
+if !options[:test] and File.exists?(new_path) and !options[:force]
+  puts "Path #{new_path} already exists, not overwriting\n"
+  exit 1
+end
+
+if options[:test]
+  puts "Syntax ok"
+
+  if options[:debug]
+    gr = KPeg::GrammarRenderer.new(grammar)
+    gr.render(STDOUT)
+  end
+  exit 0
+end
+
+
+cg = KPeg::CodeGenerator.new name, grammar
+cg.standalone = options[:standalone]
+
+File.open new_path, "w" do |f|
+  f << cg.output
+end
+
+puts "Wrote #{name} to #{new_path}"

data/doc/syntax_kpeg/ftdetect/kpeg.vim

@@ -0,0 +1 @@
+au BufNewFile,BufRead *.kpeg set filetype=kpeg

data/doc/syntax_kpeg/syntax/kpeg.vim

@@ -0,0 +1,55 @@
+" Vim syntax file
+" Language:   kpeg
+" Version:      $Revision$
+
+if version < 600
+  syntax clear
+elseif exists("b:current_syntax")
+  finish
+endif
+
+syn case match
+
+" Misc syntax.
+syn match   kpegOperator /[|*?+!\[\]]/
+syn match   kpegAssign "="
+syn match   kpegCapture /[<>]/
+syn match   kpegParen /[()]/
+
+syn match   kpegIdentifier /-|([a-zA-Z][-a-zA-Z0-9]*)/
+syn match   kpegComment /#.*$/
+syn region  kpegString start="\"" end="\"" skip="\\\\\|\\\""
+syn region  kpegRegexp start=/\// skip=/\\\// end=/\//
+
+syntax include @Ruby syntax/ruby.vim
+
+syn region  kpegCode   matchgroup=kpegCurly start=/{/ end=/}/ contains=@Ruby
+
+syn match   kpegLabel /:[a-zA-Z][-a-zA-Z0-9]*/
+
+if version >= 508 || !exists("did_c_syn_inits")
+  if version < 508
+    let did_c_syn_inits = 1
+    command -nargs=+ HiLink hi link <args>
+  else
+    command -nargs=+ HiLink hi def link <args>
+  endif
+
+  HiLink kpegRegexp Special
+  HiLink kpegNumber Number
+  HiLink kpegComment Comment
+  HiLink kpegString String
+  HiLink kpegLabel Type
+  HiLink kpegOperator Operator
+  HiLink kpegAssign Define
+  HiLink kpegCapture Keyword
+  HiLink kpegFloat Float
+  HiLink kpegIdentifier Identifier
+
+  HiLink kpegParen Delimiter
+  HiLink kpegCurly Delimiter
+
+  delcommand HiLink
+endif
+
+let b:current_syntax = "kpeg"

data/kpeg.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "kpeg/version"
+
+Gem::Specification.new do |s|
+  s.name        = "kpeg"
+  s.version     = KPeg::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Evan Phoenix"]
+  s.email       = ["evan@fallingsnow.net"]
+  s.homepage    = "https://github.com/evanphx/kpeg"
+  s.summary     = %q{Peg-based Code Generator}
+  s.description = %q{A tool for generating parsers using PEG}
+
+  rb = Dir["lib/**/*.rb"] << "bin/kpeg"
+  docs = Dir["doc/**/*"]
+
+  s.files = rb + docs + ["LICENSE", "README.md", "Rakefile", "kpeg.gemspec", "Gemfile"]
+  s.test_files    = Dir["test/**/*.rb"]
+  s.bindir = "bin"
+  s.executables = ["kpeg"]
+  s.require_paths = ["lib"]
+  s.add_development_dependency "rake"
+end

data/lib/kpeg.rb

@@ -0,0 +1,50 @@
+module KPeg
+  def self.grammar
+    g = Grammar.new
+    yield g
+    g
+  end
+
+  def self.match(str, gram)
+    scan = Parser.new(str, gram)
+    scan.parse
+  end
+
+  def self.load_grammar(file, log=false)
+    parser = KPeg::FormatParser.new File.read(file)
+    parser.parse
+
+    return parser.grammar
+  end
+
+  def self.load(file, name)
+    grammar = load_grammar(file)
+    cg = KPeg::CodeGenerator.new name, grammar
+
+    code = cg.output
+
+    warn "[Loading parser '#{name}' => #{code.size} bytes]"
+
+    Object.module_eval code
+    true
+  end
+
+  def self.compile(str, name, scope=Object)
+    parser = KPeg::FormatParser.new str
+    unless parser.parse
+      parser.raise_error
+    end
+
+    cg = KPeg::CodeGenerator.new name, parser.grammar
+
+    code = cg.output
+
+    scope.module_eval code
+    true
+  end
+end
+
+require 'kpeg/grammar'
+require 'kpeg/format_parser'
+require 'kpeg/code_generator'
+

data/lib/kpeg/code_generator.rb

@@ -0,0 +1,355 @@
+require 'kpeg/grammar_renderer'
+require 'stringio'
+
+module KPeg
+  class CodeGenerator
+    def initialize(name, gram, debug=false)
+      @name = name
+      @grammar = gram
+      @debug = debug
+      @saves = 0
+      @output = nil
+      @standalone = false
+    end
+
+    attr_accessor :standalone
+
+    def method_name(name)
+      name = name.gsub("-","_hyphen_")
+      "_#{name}"
+    end
+
+    def save
+      if @saves == 0
+        str = "_save"
+      else
+        str = "_save#{@saves}"
+      end
+
+      @saves += 1
+      str
+    end
+
+    def reset_saves
+      @saves = 0
+    end
+
+    def output_op(code, op)
+      case op
+      when Dot
+        code << "    _tmp = get_byte\n"
+      when LiteralString
+        code << "    _tmp = match_string(#{op.string.dump})\n"
+      when LiteralRegexp
+        code << "    _tmp = scan(/\\A#{op.regexp}/)\n"
+      when CharRange
+        ss = save()
+        if op.start.bytesize == 1 and op.fin.bytesize == 1
+          code << "    #{ss} = self.pos\n"
+          code << "    _tmp = get_byte\n"
+          code << "    if _tmp\n"
+
+          if op.start.respond_to? :getbyte
+            left  = op.start.getbyte 0
+            right = op.fin.getbyte 0
+          else
+            left  = op.start[0]
+            right = op.fin[0]
+          end
+
+          code << "      unless _tmp >= #{left} and _tmp <= #{right}\n"
+          code << "        self.pos = #{ss}\n"
+          code << "        _tmp = nil\n"
+          code << "      end\n"
+          code << "    end\n"
+        else
+          raise "Unsupported char range - #{op.inspect}"
+        end
+      when Choice
+        ss = save()
+        code << "\n    #{ss} = self.pos\n"
+        code << "    while true # choice\n"
+        op.ops.each_with_index do |n,idx|
+          output_op code, n
+
+          code << "    break if _tmp\n"
+          code << "    self.pos = #{ss}\n"
+          if idx == op.ops.size - 1
+            code << "    break\n"
+          end
+        end
+        code << "    end # end choice\n\n"
+      when Multiple
+        ss = save()
+        if op.min == 0 and op.max == 1
+          code << "    #{ss} = self.pos\n"
+          output_op code, op.op
+          if op.save_values
+            code << "    @result = nil unless _tmp\n"
+          end
+          code << "    unless _tmp\n"
+          code << "      _tmp = true\n"
+          code << "      self.pos = #{ss}\n"
+          code << "    end\n"
+        elsif op.min == 0 and !op.max
+          if op.save_values
+            code << "    _ary = []\n"
+          end
+
+          code << "    while true\n"
+          output_op code, op.op
+          if op.save_values
+            code << "    _ary << @result if _tmp\n"
+          end
+          code << "    break unless _tmp\n"
+          code << "    end\n"
+          code << "    _tmp = true\n"
+
+          if op.save_values
+            code << "    @result = _ary\n"
+          end
+
+        elsif op.min == 1 and !op.max
+          code << "    #{ss} = self.pos\n"
+          if op.save_values
+            code << "    _ary = []\n"
+          end
+          output_op code, op.op
+          code << "    if _tmp\n"
+          if op.save_values
+            code << "      _ary << @result\n"
+          end
+          code << "      while true\n"
+          code << "    "
+          output_op code, op.op
+          if op.save_values
+            code << "        _ary << @result if _tmp\n"
+          end
+          code << "        break unless _tmp\n"
+          code << "      end\n"
+          code << "      _tmp = true\n"
+          if op.save_values
+            code << "      @result = _ary\n"
+          end
+          code << "    else\n"
+          code << "      self.pos = #{ss}\n"
+          code << "    end\n"
+        else
+          code << "    #{ss} = self.pos\n"
+          code << "    _count = 0\n"
+          code << "    while true\n"
+          code << "  "
+          output_op code, op.op
+          code << "      if _tmp\n"
+          code << "        _count += 1\n"
+          code << "        break if _count == #{op.max}\n"
+          code << "      else\n"
+          code << "        break\n"
+          code << "      end\n"
+          code << "    end\n"
+          code << "    if _count >= #{op.min}\n"
+          code << "      _tmp = true\n"
+          code << "    else\n"
+          code << "      self.pos = #{ss}\n"
+          code << "      _tmp = nil\n"
+          code << "    end\n"
+        end
+
+      when Sequence
+        ss = save()
+        code << "\n    #{ss} = self.pos\n"
+        code << "    while true # sequence\n"
+        op.ops.each_with_index do |n, idx|
+          output_op code, n
+
+          if idx == op.ops.size - 1
+            code << "    unless _tmp\n"
+            code << "      self.pos = #{ss}\n"
+            code << "    end\n"
+            code << "    break\n"
+          else
+            code << "    unless _tmp\n"
+            code << "      self.pos = #{ss}\n"
+            code << "      break\n"
+            code << "    end\n"
+          end
+        end
+        code << "    end # end sequence\n\n"
+      when AndPredicate
+        ss = save()
+        code << "    #{ss} = self.pos\n"
+        if op.op.kind_of? Action
+          code << "    _tmp = begin; #{op.op.action}; end\n"
+        else
+          output_op code, op.op
+        end
+        code << "    self.pos = #{ss}\n"
+      when NotPredicate
+        ss = save()
+        code << "    #{ss} = self.pos\n"
+        if op.op.kind_of? Action
+          code << "    _tmp = begin; #{op.op.action}; end\n"
+        else
+          output_op code, op.op
+        end
+        code << "    _tmp = _tmp ? nil : true\n"
+        code << "    self.pos = #{ss}\n"
+      when RuleReference
+        code << "    _tmp = apply(:#{method_name op.rule_name})\n"
+      when InvokeRule
+        if op.arguments
+          code << "    _tmp = #{method_name op.rule_name}#{op.arguments}\n"
+        else
+          code << "    _tmp = #{method_name op.rule_name}()\n"
+        end
+      when ForeignInvokeRule
+        if op.arguments
+          code << "    _tmp = @_grammar_#{op.grammar_name}.external_invoke(self, :#{method_name op.rule_name}, #{op.arguments[1..-2]})\n"
+        else
+          code << "    _tmp = @_grammar_#{op.grammar_name}.external_invoke(self, :#{method_name op.rule_name})\n"
+        end
+      when Tag
+        if op.tag_name and !op.tag_name.empty?
+          output_op code, op.op
+          code << "    #{op.tag_name} = @result\n"
+        else
+          output_op code, op.op
+        end
+      when Action
+        code << "    @result = begin; "
+        code << op.action << "; end\n"
+        if @debug
+          code << "    puts \"   => \" #{op.action.dump} \" => \#{@result.inspect} \\n\"\n"
+        end
+        code << "    _tmp = true\n"
+      when Collect
+        code << "    _text_start = self.pos\n"
+        output_op code, op.op
+        code << "    if _tmp\n"
+        code << "      text = get_text(_text_start)\n"
+        code << "    end\n"
+      else
+        raise "Unknown op - #{op.class}"
+      end
+
+    end
+
+    def standalone_region(path)
+      cp = File.read(path)
+      start = cp.index("# STANDALONE START")
+      fin = cp.index("# STANDALONE END")
+
+      return nil unless start and fin
+      cp[start..fin]
+    end
+
+    def output
+      return @output if @output
+      if @standalone
+        code = "class #{@name}\n"
+
+        unless cp = standalone_region(
+                    File.expand_path("../compiled_parser.rb", __FILE__))
+
+          puts "Standalone failure. Check compiler_parser.rb for proper boundary comments"
+          exit 1
+        end
+
+        unless pp = standalone_region(
+                    File.expand_path("../position.rb", __FILE__))
+          puts "Standalone failure. Check position.rb for proper boundary comments"
+        end
+
+        cp.gsub!(/include Position/, pp)
+        code << cp << "\n"
+      else
+        code =  "require 'kpeg/compiled_parser'\n\n"
+        code << "class #{@name} < KPeg::CompiledParser\n"
+      end
+
+      @grammar.setup_actions.each do |act|
+        code << "\n#{act.action}\n\n"
+      end
+
+      fg = @grammar.foreign_grammars
+
+      if fg.empty?
+        if @standalone
+          code << "  def setup_foreign_grammar; end\n"
+        end
+      else
+        code << "  def setup_foreign_grammar\n"
+        @grammar.foreign_grammars.each do |name, gram|
+          code << "    @_grammar_#{name} = #{gram}.new(nil)\n"
+        end
+        code << "  end\n"
+      end
+
+      render = GrammarRenderer.new(@grammar)
+
+      renderings = {}
+
+      @grammar.rule_order.each do |name|
+        reset_saves
+
+        rule = @grammar.rules[name]
+        io = StringIO.new
+        render.render_op io, rule.op
+
+        rend = io.string
+        rend.gsub! "\n", " "
+
+        renderings[name] = rend
+
+        code << "\n"
+        code << "  # #{name} = #{rend}\n"
+
+        if rule.arguments
+          code << "  def #{method_name name}(#{rule.arguments.join(',')})\n"
+        else
+          code << "  def #{method_name name}\n"
+        end
+
+        if @debug
+          code << "    puts \"START #{name} @ \#{show_pos}\\n\"\n"
+        end
+
+        output_op code, rule.op
+        if @debug
+          code << "    if _tmp\n"
+          code << "      puts \"   OK #{name} @ \#{show_pos}\\n\"\n"
+          code << "    else\n"
+          code << "      puts \" FAIL #{name} @ \#{show_pos}\\n\"\n"
+          code << "    end\n"
+        end
+
+        code << "    set_failed_rule :#{method_name name} unless _tmp\n"
+        code << "    return _tmp\n"
+        code << "  end\n"
+      end
+
+      code << "\n  Rules = {}\n"
+      @grammar.rule_order.each do |name|
+        rule = @grammar.rules[name]
+
+        rend = GrammarRenderer.escape renderings[name], true
+        code << "  Rules[:#{method_name name}] = rule_info(\"#{name}\", \"#{rend}\")\n"
+      end
+
+      code << "end\n"
+      @output = code
+    end
+
+    def make(str)
+      m = Module.new
+      m.module_eval output
+
+      cls = m.const_get(@name)
+      cls.new(str)
+    end
+
+    def parse(str)
+      make(str).parse
+    end
+  end
+end