kpeg 0.8.5 → 0.9.0

data/.autotest

@@ -0,0 +1,10 @@
+require 'autotest/restart'
+
+Autotest.add_hook :run_command do |at|
+  system "#{Gem.ruby} -rubygems #{Gem.bin_path 'rake', 'rake'} parser"
+end
+
+Autotest.add_hook :initialize do |at|
+  at.testlib = 'minitest/autorun'
+end
+

data/Gemfile

@@ -1,4 +1,12 @@
-source "http://rubygems.org"
+# -*- ruby -*-
 
-# Specify your gem's dependencies in callisto.gemspec
-gemspec
+# DO NOT EDIT THIS FILE. Instead, edit Rakefile, and run `rake bundler:gemfile`.
+
+source :gemcutter
+
+
+gem "minitest", "~>2.11", :group => [:development, :test]
+gem "rdoc", "~>3.10", :group => [:development, :test]
+gem "hoe", "~>2.15", :group => [:development, :test]
+
+# vim: syntax=ruby

data/History.txt

@@ -0,0 +1,21 @@
+=== 1.0 / 2012-04-06
+
+* Minor enhancements
+  * Added arbitrary directives to the kpeg grammar
+
+      %% directive_name { ... }
+
+  * Added header and footer directives to the kpeg code formatter.  These
+    appear above and below all other output, respectively:
+
+      %% header {
+       # coding: UTF-8
+      }
+
+      [... your grammar ...]
+
+      %% footer {
+        require 'some/subclass'
+      }
+  * Switched to minitest
+  * Switched to hoe

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2011, Evan Phoenix
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the <organization> nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/Manifest.txt

@@ -0,0 +1,47 @@
+.autotest
+Gemfile
+History.txt
+LICENSE
+Manifest.txt
+README.rdoc
+Rakefile
+bin/kpeg
+examples/calculator/calculator.kpeg
+examples/calculator/calculator.rb
+examples/foreign_reference/literals.kpeg
+examples/foreign_reference/matcher.kpeg
+examples/foreign_reference/matcher.rb
+examples/lua_string/driver.rb
+examples/lua_string/lua_string.kpeg
+examples/lua_string/lua_string.kpeg.rb
+examples/phone_number/README.md
+examples/phone_number/phone_number.kpeg
+examples/phone_number/phone_number.rb
+examples/upper/README.md
+examples/upper/upper.kpeg
+examples/upper/upper.rb
+kpeg.gemspec
+lib/hoe/kpeg.rb
+lib/kpeg.rb
+lib/kpeg/code_generator.rb
+lib/kpeg/compiled_parser.rb
+lib/kpeg/format_parser.kpeg
+lib/kpeg/format_parser.rb
+lib/kpeg/grammar.rb
+lib/kpeg/grammar_renderer.rb
+lib/kpeg/match.rb
+lib/kpeg/parser.rb
+lib/kpeg/position.rb
+lib/kpeg/string_escape.kpeg
+lib/kpeg/string_escape.rb
+test/inputs/comments.kpeg
+test/test_file_parser_roundtrip.rb
+test/test_gen_calc.rb
+test/test_kpeg.rb
+test/test_kpeg_code_generator.rb
+test/test_kpeg_compiled_parser.rb
+test/test_kpeg_format.rb
+test/test_kpeg_grammar_renderer.rb
+test/test_left_recursion.rb
+vim/syntax_kpeg/ftdetect/kpeg.vim
+vim/syntax_kpeg/syntax/kpeg.vim

data/README.rdoc

@@ -0,0 +1,222 @@
+= kpeg
+
+home :: https://github.com/evanphx/kpeg
+bugs :: https://github.com/evanphx/kpeg/issues
+
+== Description
+
+KPeg is a simple PEG library for Ruby. It provides an API as well as native
+grammar to build the grammar.
+
+KPeg strives to provide a simple, powerful API without being too exotic.
+
+KPeg supports direct left recursion of rules via the
+{OMeta memoization}[http://www.vpri.org/pdf/tr2008003_experimenting.pdf] trick.
+
+== Writing your first grammar
+
+=== Setting up your grammar
+
+All grammars start with with the class/module name that will be your parser
+
+  %% name = Example::Parser
+
+After that a block of ruby code can be defined that will be added into the
+class body of your parser. Attributes that are defined in this block can be
+accessed within your parser as instance variables. Methods can also be defined
+in this block and used in action blocks as well.
+
+  %% {
+    attr_accessor :something_cool
+    
+    def something_awesome
+      # do something awesome
+    end
+  }
+
+=== Defining literals
+
+Literals are static declarations of characters or regular expressions designed for reuse in the grammar. These can be constants or variables. Literals can take strings, regular expressions or character ranges
+
+  ALPHA = /[A-Za-z]/
+  DIGIT = /[0-9]/
+  period = "."
+  string = "a string"
+  regex = /(regexs?)+/
+  char_range = [b-t]
+
+Literals can also accept multiple definitions
+
+  vowel = "a" | "e" | "i" | "o" | "u"
+  alpha = /[A-Z]/ | /[a-z]/
+
+=== Defining Rules for Values
+
+Before you can start parsing a string you will need to define rules that you
+will use to accept or reject that string. There are many different types of
+rules available in kpeg 
+
+The most basic of these rules is a string capture
+
+  alpha = < /[A-Za-z]/ > { text }
+
+While this looks very much like the ALPHA literal defined above it differs in
+one important way, the text captured by the rule defined between the < and >
+symbols will be set as the text variable in block that follows. You can also
+explicitly define the variable that you would like but only with existing
+rules or literals.
+
+  letter = alpha:a { a }
+
+Additionally blocks can return true or false values based upon an expression
+within the block. To return true if a test passes do the following:
+
+  match_greater_than_10 = < num:n > &{ n > 10 }
+
+To test and return a false value if the test passes do the following:
+
+  do_not_match_greater_than_10 = < num:n > !{ n > 10 }
+
+Rules can also act like functions and take parameters. An example of this is
+lifted from the {Email List
+Validator}[https://github.com/larb/email_address_validator], where an ascii
+value is passed in and the character is evaluated against it returning a true
+if it matches
+
+  d(num) = <.> &{ text[0] == num }
+
+Rules support some regular expression syntax for matching
+
+* maybe ?
+* many +
+* kleene *
+* groupings ()
+
+Examples:
+
+  letters = alpha+
+  words = alpha+ space* period?
+  sentence = (letters+ | space+)+
+
+Kpeg also allows a rule to define the acceptable number of matches in the form
+of a range. In regular expressions this is often denoted with syntax like
+{0,3}. Kpeg uses this syntax to accomplish match ranges [min, max].
+
+  matches_3_to_5_times = letter[3,5]
+  matches_3_to_any_times = letter[3,*]
+
+=== Defining Actions
+
+Illustrated above in some of the examples, kpeg allows you to perform actions
+based upon a match that are described in block provided or in the rule
+definition itself.
+
+  num = /[1-9][0-9]*/
+  sum = < num:n1 "+" num:n2 > { n1 + n2 }
+
+As of version 0.8 an alternate syntax has been added for calling defined
+methods as actions.
+
+  %% {
+    def add(n1, n2){
+      n1 + n2
+    }
+  }
+  num = /[1-9][0-9]*/
+  sum = < num:n1 "+" num:n2 > ~add(n1, n2)
+
+=== Referencing an external grammar
+
+Kpeg allows you to run a rule that is defined in an external grammar. This is
+useful if there is a defined set of rules that you would like to reuse in
+another parser. To do this, create your grammar and generate a parser using
+the kpeg command line tool.
+
+  kpeg literals.kpeg
+
+Once you have the generated parser, include that file into your new grammar
+
+  %{
+    require "literals.kpeg.rb"
+  }
+
+Then create a variable to hold to foreign interface and pass it the class name
+of your parser. In this case my parser class name is Literal 
+
+  %foreign_grammer = Literal
+
+You can then use rules defined in the foreign grammar in the local grammar
+file like so
+
+  sentence = (%foreign_grammer.alpha %foreign_grammer.space*)+
+             %foreign_grammer.period
+
+=== Comments
+
+Kpeg allows comments to be added to the grammar file by using the # symbol
+
+  # This is a comment in my grammar
+
+== Generating and running your parser
+
+Before you can generate your parser you will need to define a root rule. This
+will be the first rule run against the string provided to the parser
+
+  root = sentence
+
+To generate the parser run the kpeg command with the kpeg file(s) as an
+argument. This will generate a ruby file with the same name as your grammar
+file.
+
+  kpeg example.kpeg
+
+Include your generated parser file into an application that you want to use
+the parser in and run it. Create a new instance of the parser and pass in the
+string you want to evaluate. When parse is called on the parser instance it
+will return a true if the sting is matched, or false if it doesn't. 
+
+  require "example.kpeg.rb"
+
+  parser = Example::Parser.new(string_to_evaluate)
+  parser.parse
+
+== Shortcuts and other techniques
+
+Per vito, you can get the current line or current column in the following way 
+
+  line = { current_line }
+  column = { current_column }
+  foo = line:line ... { # use line here }
+
+== AST Generation
+
+As of Kpeg 0.8 a parser can now generate an AST. To define an AST node use the
+following syntax
+
+  %% assign = ast Assignment(name, value)
+
+Once you have a defined AST node, it can be used in your grammar like so
+
+  assignment = identifier:i space* = space* value:v ~assign(i,v)
+
+This will create a new Assign node that you can add into your AST.
+
+For a good example of usage check out Talon[https://github.com/evanphx/talon]
+
+== Examples
+
+There are several examples available in the /examples directory. The upper
+parser has a readme with a step by step description of the grammar.
+
+== Projects using kpeg
+
+Dang[https://github.com/veganstraightedge/dang]
+
+{Email Address Validator}[https://github.com/larb/email_address_validator]
+
+Callisto[https://github.com/dwaite/Callisto]
+
+Doodle[https://github.com/vito/doodle]
+
+Kanbanpad[https://kanbanpad.com] (uses kpeg for parsing of the 'enter
+something' bar)

data/Rakefile

@@ -1,15 +1,21 @@
-require 'rake/testtask'
+# -*- ruby -*-
 
-$:.unshift "lib"
+require 'rubygems'
+require 'hoe'
 
-task :default => :test
+Hoe.plugin :bundler
+Hoe.plugin :gemspec
+Hoe.plugin :git
+Hoe.plugin :minitest
+Hoe.plugin :travis
 
-desc "Run tests"
-Rake::TestTask.new do |t|
-  t.test_files = FileList['test/test*.rb']
-  t.verbose = true
+Hoe.spec 'kpeg' do
+  self.readme_file = "README.rdoc"
+  developer 'Evan Phoenix', 'evan@fallingsnow.net'
 end
 
+task :test => :parser
+
 task :grammar do
   require 'kpeg'
   require 'kpeg/format'
@@ -19,8 +25,14 @@ task :grammar do
   gr.render(STDOUT)
 end
 
-desc "rebuild parser"
-task :parser do
-  sh "ruby -Ilib bin/kpeg -o lib/kpeg/string_escape.rb -f lib/kpeg/string_escape.kpeg"
-  sh "ruby -Ilib bin/kpeg -o lib/kpeg/format_parser.rb -s -f lib/kpeg/format.kpeg"
+rule ".rb" => ".kpeg" do |t|
+  ruby "-Ilib bin/kpeg -s -o #{t.name} -f #{t.source}"
 end
+
+desc "build the parser"
+task :parser => %w[
+  lib/kpeg/string_escape.rb
+  lib/kpeg/format_parser.rb
+]
+
+# vim: syntax=ruby

data/bin/kpeg

@@ -119,8 +119,10 @@ end
 cg = KPeg::CodeGenerator.new name, grammar
 cg.standalone = options[:standalone]
 
-File.open new_path, "w" do |f|
-  f << cg.output
+output = cg.output
+
+open new_path, "w" do |io|
+  io << output
 end
 
 puts "Wrote #{name} to #{new_path}"

data/examples/calculator/calculator.kpeg

@@ -0,0 +1,17 @@
+%% name = Calculator
+
+%% {
+  attr_accessor :result
+}
+
+space = " "
+- = space*
+num = < /[1-9][0-9]*/ > { text.to_i }
+
+term = term:t1 - "+" - term:t2 { t1 + t2 }
+     | term:t1 - "-" - termLt2 { t1 - t2 }
+     | fact
+fact = fact:f1 - "*" - fact:f2 { f1 * f2 }
+     | fact:f1 - "/" - fact:f2 { f1 / f2 }
+     | num
+root = term:t { @result = t }

data/examples/calculator/calculator.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require "./calculator.kpeg"
+
+parser = Calculator.new("1 + 2 * 3")
+if parser.parse
+  puts parser.result
+end

data/examples/foreign_reference/literals.kpeg

@@ -0,0 +1,5 @@
+%% name = Literal
+
+period = "."
+space = " "
+alpha = /[A-Za-z]/