kpeg 0.8.5 → 0.9.0

data/examples/phone_number/README.md

@@ -0,0 +1,3 @@
+# Phone Number Parser
+
+A string is parsed to determine if it is a valid phone number. A phone number in the following format (888) 555-0000 will be available using the resolved_number method if the string successfully matches

data/examples/phone_number/phone_number.kpeg

@@ -0,0 +1,20 @@
+%% name = PhoneNumber
+
+%% {
+	attr_accessor :phone_number
+}
+digit = [0-9]
+space = " "
+dash = "-"
+LP = "("
+RP = ")"
+
+country_code = < digit > { text }
+area_code = < digit[3] > { text }
+prefix = < digit[3] > { text }
+suffix = < digit[4] > { text }
+
+phone_number = LP? area_code:ac RP? space* prefix:p space* dash? space* suffix:s space* { "(#{ac}) #{p}-#{s}" } 
+
+root = phone_number:pn { @phone_number = pn }
+		| country_code:c space* phone_number:pn { @phone_number = "+#{c} #{pn}" }

data/examples/phone_number/phone_number.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require "./phone_number.kpeg.rb"
+
+parser = PhoneNumber.new("8888888888")
+puts parser.parse
+puts parser.phone_number

data/examples/upper/README.md

@@ -0,0 +1,83 @@
+# Upper Parser
+
+A parser that matches a string with alpha characters, spaces and a period and returns the string in upper case.
+
+## Grammar
+
+Name of the class that will be used to do the parsing
+  
+    %% name = Upper
+
+A variable that I want to store the converted text for accessing later
+    
+    %% {
+        attr_accessor :output
+    }
+
+My literals
+    
+    period = "."
+    space = " "
+
+A rule that states that all characters that match the regex [A-Za-z] should be returned uppercase
+
+    alpha = < /[A-Za-z]/ > { text.upcase }
+
+My rules that defines a word, it consists of three different cases, first that a word is an alpha followed by another word. If this matches return the alpha and the word that follows.
+
+
+    word = alpha:a word:w { "#{a}#{w}" }
+
+This rule states that a word can be an alpha followed by a space. If this matches return an alpha followed by a space
+
+    | alpha:a space+ { "#{a} "}
+      
+This rule states that a word can consist of just an alpha. If this matches just return the alpha
+      
+    | alpha:a { a }
+
+My rules that defines a sentence. The first states that a sentence consists of a word followed by a sentence. If this matches return the word followed by the sentence.
+
+    sentence = < word:w sentence:s > { "#{w}#{s}" }
+
+This rule states that a sentence can just be a word. If this matches just return the word.
+    
+    | word:w { w }
+
+My rules that define a document. The first rule states that a document can be a sentence followed by a period that may have space followed by another document. If this matches return the sentence followed by a period with a space followed by the document.
+
+    document = sentence:s period space* document:d  { "#{s}. #{d}" }
+
+This rule states that a document can be a sentence followed by a period. If this matches return the sentence followed by a period.
+          
+    | sentence:s period { "#{s}." }
+          
+This rule states that a document can just be a sentence. If this matches just return the sentence.
+          
+    | sentence:s { s }
+
+The root node it the first rule evaluated, is it essentially the starting point for your grammar. If the string provided can successfully be matched by the grammar provided store the returned document in the @output variable.
+
+    root = document:d { @output = d }
+    
+## Generate the parser
+
+To generate the parser make sure you have kpeg installed and run the following command (you may have to remove upper.kpeg.rb if it was previously generated)
+
+    kpeg upper.kpeg
+    
+## Run the parser
+
+To run the parser run the following
+
+    ruby upper.rb
+
+## Accepted Strings
+
++ a lower case string. Another lower case string.
++ A LOWER CASE STRING. ANOTHER LOWER CASE STRING.
++ a  string     with   lots  of    spaces.
+
+## Not accepted strings (there are tons)
+
+Anything that doesn't stick to spaces and periods, very brittle but it is a simple example

data/examples/upper/upper.kpeg

@@ -0,0 +1,24 @@
+%% name = Upper
+
+
+%% {
+    attr_accessor :output
+}
+
+period = "."
+space = " "
+alpha = < /[A-Za-z]/ > { text.upcase }
+
+word = alpha:a word:w { "#{a}#{w}" }
+      | alpha:a space { "#{a} "}    
+      | alpha:a { a }
+
+sentence = word:w sentence:s { "#{w}#{s}" }
+          | word:w { w }
+
+document = sentence:s period space* document:d  { "#{s}. #{d}" }
+          | sentence:s period { "#{s}." }
+          | sentence:s { "#{s}" }
+
+root = document:d { @output = d }
+

data/examples/upper/upper.rb

@@ -0,0 +1,9 @@
+# Make sure you have the kpeg gem installed
+require 'rubygems'
+# To generate the upper.kpeg file run kpeg upper.kpeg
+require "./upper.kpeg.rb" # Require the generated parser
+
+parser = Upper.new("a lower case string. Another lower case string.")
+if parser.parse
+  puts parser.output
+end

data/kpeg.gemspec

@@ -1,24 +1,42 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "kpeg/version"
 
 Gem::Specification.new do |s|
-  s.name        = "kpeg"
-  s.version     = KPeg::VERSION
-  s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Evan Phoenix"]
-  s.email       = ["evan@fallingsnow.net"]
-  s.homepage    = "https://github.com/evanphx/kpeg"
-  s.summary     = %q{Peg-based Code Generator}
-  s.description = %q{A tool for generating parsers using PEG}
+  s.name = "kpeg"
+  s.version = "0.8.5.20120306163408"
 
-  rb = Dir["lib/**/*.rb"] << "bin/kpeg"
-  docs = Dir["doc/**/*"]
-
-  s.files = rb + docs + ["README.md", "Rakefile", "kpeg.gemspec", "Gemfile"]
-  s.test_files    = Dir["test/**/*.rb"]
-  s.bindir = "bin"
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Eric Hodel"]
+  s.cert_chain = ["/Users/drbrain/.gem/gem-public_cert.pem"]
+  s.date = "2012-03-07"
+  s.description = "KPeg is a simple PEG library for Ruby. It provides an API as well as native\ngrammar to build the grammar.\n\nKPeg strives to provide a simple, powerful API without being too exotic.\n\nKPeg supports direct left recursion of rules via the\n{OMeta memoization}[http://www.vpri.org/pdf/tr2008003_experimenting.pdf] trick."
+  s.email = ["drbrain@segment7.net"]
   s.executables = ["kpeg"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.rdoc"]
+  s.files = [".autotest", "Gemfile", "Gemfile.lock", "History.txt", "LICENSE", "Manifest.txt", "README.rdoc", "Rakefile", "bin/kpeg", "examples/calculator/calculator.kpeg", "examples/calculator/calculator.rb", "examples/foreign_reference/literals.kpeg", "examples/foreign_reference/matcher.kpeg", "examples/foreign_reference/matcher.rb", "examples/lua_string/driver.rb", "examples/lua_string/lua_string.kpeg", "examples/lua_string/lua_string.kpeg.rb", "examples/phone_number/README.md", "examples/phone_number/phone_number.kpeg", "examples/phone_number/phone_number.rb", "examples/upper/README.md", "examples/upper/upper.kpeg", "examples/upper/upper.rb", "kpeg.gemspec", "lib/kpeg.rb", "lib/kpeg/code_generator.rb", "lib/kpeg/compiled_parser.rb", "lib/kpeg/format.kpeg", "lib/kpeg/format_parser.rb", "lib/kpeg/grammar.rb", "lib/kpeg/grammar_renderer.rb", "lib/kpeg/match.rb", "lib/kpeg/parser.rb", "lib/kpeg/position.rb", "lib/kpeg/string_escape.kpeg", "lib/kpeg/string_escape.rb", "lib/kpeg/version.rb", "test/inputs/comments.kpeg", "test/test_file_parser_roundtrip.rb", "test/test_gen_calc.rb", "test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_grammar_renderer.rb", "test/test_left_recursion.rb", "vim/syntax_kpeg/ftdetect/kpeg.vim", "vim/syntax_kpeg/syntax/kpeg.vim", ".gemtest"]
+  s.homepage = "https://github.com/evanphx/kpeg"
+  s.rdoc_options = ["--main", "README.rdoc"]
   s.require_paths = ["lib"]
-  s.add_development_dependency "rake"
+  s.rubyforge_project = "kpeg"
+  s.rubygems_version = "1.8.12"
+  s.signing_key = "/Users/drbrain/.gem/gem-private_key.pem"
+  s.summary = "KPeg is a simple PEG library for Ruby"
+  s.test_files = ["test/test_file_parser_roundtrip.rb", "test/test_gen_calc.rb", "test/test_kpeg.rb", "test/test_kpeg_code_generator.rb", "test/test_kpeg_compiled_parser.rb", "test/test_kpeg_format.rb", "test/test_kpeg_grammar_renderer.rb", "test/test_left_recursion.rb"]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<minitest>, ["~> 2.11"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.10"])
+      s.add_development_dependency(%q<hoe>, ["~> 2.15"])
+    else
+      s.add_dependency(%q<minitest>, ["~> 2.11"])
+      s.add_dependency(%q<rdoc>, ["~> 3.10"])
+      s.add_dependency(%q<hoe>, ["~> 2.15"])
+    end
+  else
+    s.add_dependency(%q<minitest>, ["~> 2.11"])
+    s.add_dependency(%q<rdoc>, ["~> 3.10"])
+    s.add_dependency(%q<hoe>, ["~> 2.15"])
+  end
 end

data/lib/hoe/kpeg.rb

@@ -0,0 +1,94 @@
+##
+# Kpeg plugin for hoe.
+#
+# === Tasks Provided:
+#
+# parser            :: Generate parsers for all .kpeg files in your manifest
+# .kpeg -> .rb rule :: Generate a parser using kpeg.
+#
+# NOTE: This plugin is derived from the Hoe::Racc and used under the MIT
+# license:
+#
+# Copyright (c) Ryan Davis, seattle.rb
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module Hoe::Kpeg
+
+  ##
+  # Optional: Defines what tasks need to generate parsers first.
+  #
+  # Defaults to [:multi, :test, :check_manifest]
+  #
+  # If you have extra tasks that require your parser to be built, add their
+  # names here in your hoe spec. eg:
+  #
+  #    kpeg_tasks << :debug
+
+  attr_accessor :kpeg_tasks
+
+  ##
+  # Optional: Defines what flags to use for kpeg. default: "-s -v"
+
+  attr_accessor :kpeg_flags
+
+  ##
+  # Initialize variables for kpeg plugin.
+
+  def initialize_kpeg
+    self.kpeg_tasks = [:multi, :test, :check_manifest]
+
+    # -v = verbose
+    # -s = parser does not require runtime
+    self.kpeg_flags ||= "-s -v"
+
+    dependency 'kpeg', '~> 0.9', :development
+  end
+
+  ##
+  # Define tasks for kpeg plugin
+
+  def define_kpeg_tasks
+    kpeg_files   = self.spec.files.find_all { |f| f =~ /\.kpeg$/ }
+
+    parser_files = kpeg_files.map { |f| f.sub(/\.kpeg$/, ".rb") }
+
+    self.clean_globs += parser_files
+
+    rule ".rb" => ".kpeg" do |t|
+      kpeg = Gem.bin_path "kpeg", "kpeg"
+
+      begin
+        ruby "-rubygems #{kpeg} #{kpeg_flags} -o #{t.name} #{t.source}"
+      rescue
+        abort "need kpeg, please run rake check_extra_deps"
+      end
+    end
+
+    desc "build the parser" unless parser_files.empty?
+    task :parser
+
+    task :parser => parser_files
+
+    kpeg_tasks.each do |t|
+      task t => :parser
+    end
+  end
+end

data/lib/kpeg.rb

@@ -1,4 +1,7 @@
 module KPeg
+
+  VERSION = "0.9.0"
+
   def self.grammar
     g = Grammar.new
     yield g

data/lib/kpeg/code_generator.rb

@@ -323,8 +323,16 @@ module KPeg
 
     def output
       return @output if @output
+
+      code = []
+
+      if header = @grammar.directives['header']
+        code << header.action.strip
+        code << "\n"
+      end
+
       if @standalone
-        code = "class #{@name}\n"
+        code << "class #{@name}\n"
 
         unless cp = standalone_region(
                     File.expand_path("../compiled_parser.rb", __FILE__))
@@ -341,7 +349,7 @@ module KPeg
         cp.gsub!(/include Position/, pp)
         code << cp << "\n"
       else
-        code =  "require 'kpeg/compiled_parser'\n\n"
+        code << "require 'kpeg/compiled_parser'\n\n"
         code << "class #{@name} < KPeg::CompiledParser\n"
       end
 
@@ -417,7 +425,12 @@ module KPeg
       end
 
       code << "end\n"
-      @output = code
+
+      if footer = @grammar.directives['footer']
+        code << footer.action
+      end
+
+      @output = code.join
     end
 
     def make(str)

data/lib/kpeg/compiled_parser.rb

@@ -183,32 +183,24 @@ module KPeg
       end
     end
 
-    class LeftRecursive
-      def initialize(detected=false)
-        @detected = detected
-      end
-
-      attr_accessor :detected
-    end
-
     class MemoEntry
       def initialize(ans, pos)
         @ans = ans
         @pos = pos
-        @uses = 1
         @result = nil
+        @set = false
+        @left_rec = false
       end
 
-      attr_reader :ans, :pos, :uses, :result
-
-      def inc!
-        @uses += 1
-      end
+      attr_reader :ans, :pos, :result, :set
+      attr_accessor :left_rec
 
       def move!(ans, pos, result)
         @ans = ans
         @pos = pos
         @result = result
+        @set = true
+        @left_rec = false
       end
     end
 
@@ -236,12 +228,10 @@ module KPeg
     def apply_with_args(rule, *args)
       memo_key = [rule, args]
       if m = @memoizations[memo_key][@pos]
-        m.inc!
-
         prev = @pos
         @pos = m.pos
-        if m.ans.kind_of? LeftRecursive
-          m.ans.detected = true
+        if !m.set
+          m.left_rec = true
           return nil
         end
 
@@ -249,18 +239,19 @@ module KPeg
 
         return m.ans
       else
-        lr = LeftRecursive.new(false)
-        m = MemoEntry.new(lr, @pos)
+        m = MemoEntry.new(nil, @pos)
         @memoizations[memo_key][@pos] = m
         start_pos = @pos
 
         ans = __send__ rule, *args
 
+        lr = m.left_rec
+
         m.move! ans, @pos, @result
 
         # Don't bother trying to grow the left recursion
         # if it's failing straight away (thus there is no seed)
-        if ans and lr.detected
+        if ans and lr
           return grow_lr(rule, args, start_pos, m)
         else
           return ans
@@ -272,12 +263,10 @@ module KPeg
 
     def apply(rule)
       if m = @memoizations[rule][@pos]
-        m.inc!
-
         prev = @pos
         @pos = m.pos
-        if m.ans.kind_of? LeftRecursive
-          m.ans.detected = true
+        if !m.set
+          m.left_rec = true
           return nil
         end
 
@@ -285,18 +274,19 @@ module KPeg
 
         return m.ans
       else
-        lr = LeftRecursive.new(false)
-        m = MemoEntry.new(lr, @pos)
+        m = MemoEntry.new(nil, @pos)
         @memoizations[rule][@pos] = m
         start_pos = @pos
 
         ans = __send__ rule
 
+        lr = m.left_rec
+
         m.move! ans, @pos, @result
 
         # Don't bother trying to grow the left recursion
         # if it's failing straight away (thus there is no seed)
-        if ans and lr.detected
+        if ans and lr
           return grow_lr(rule, nil, start_pos, m)
         else
           return ans