peggy 0.1.0

data/lib/peg.rb

@@ -0,0 +1,68 @@
+require 'rubygems'
+require 'builder'
+# require File.join(File.dirname(__FILE__), 'builder')
+
+module Peggy
+
+  # Implements the Parser Expression Grammar (PEG), one of several grammars supported.
+  class PEG < Builder
+
+    def initialize
+      super
+      build
+    end
+
+  private
+  
+    def build
+      # Hierarchical syntax
+      grammar{each{spacing; some{definition}; eof}}
+      definition{each{identifier; left_arrow; expression}}
+      expression{each{sequence; many{each{slash; sequence}}}}
+      sequence{many{prefix}}
+      prefix{each{opt{one{peek; disallow}}; suffix}}
+      suffix{each{primary; opt{one{question; star; plus}}}}
+      primary{one{each{identifier; neg{left_arrow}}
+        each{lparen; expression; rparen}
+        literal; klass; dot}
+      }
+      # Lexical syntax
+      identifier{each{ident_start; many{ident_cont}; spacing}}
+      ident_start{lit /[a-zA-Z_]/}
+      ident_cont{one{ident_start; lit /[0-9]/}}
+      literal{one{
+        each{quote; many{each{neg{quote}; char}}; quote; spacing}
+        each{quotes; many{each{neg{quotes}; char}}; quotes; spacing}
+      }}
+      klass{each{lit '['; many{each{neg{lit ']'}; range}}; lit ']'; spacing}}
+      range{one{each{char; lit '-'; char}; char}}
+      char{lit /\\([nrt'"\[\]\\]|[0-2][0-7][0-7]|[0-7][0-7]?)|[^\\]/}
+      #char{one{
+      #  each{lit '\\'; one{
+      #    quote
+      #    quotes
+      #    lit /[nrt\[\]\\]/
+      #    lit /[0-2][0-7][0-7]/
+      #    lit /[0-7][0-7]?/
+      #  }}
+      #  lit /[^\\]/
+      #}}
+      left_arrow{each{lit '<-'; spacing}}
+      slash{each{lit '/'; spacing}}
+      peek{each{lit '&'; spacing}}
+      disallow{each{lit '!'; spacing}}
+      question{each{lit '?'; spacing}}
+      star{each{lit '*'; spacing}}
+      plus{each{lit '+'; spacing}}
+      lparen{each{lit '('; spacing}}
+      rparen{each{lit ')'; spacing}}
+      dot{each{lit '.'; spacing}}
+      quote{lit "'"}
+      quotes{lit '"'}
+      spacing{many{one{space; comment}}}
+      comment{each{lit '#'; many{each{neg{eol}; lit /./}}; eol}}
+      space{one{lit ' '; lit "\t"; eol}}
+      eol{one{lit "\r\n"; lit "\n"; lit "\r"}}
+    end
+  end
+end # Peggy

data/lib/peggy.rb

@@ -0,0 +1,50 @@
+require 'rubygems'
+require 'builder'
+#require File.join(File.dirname(__FILE__), 'builder')
+
+module Peggy
+
+  # Implements a variation to the Parser Expression Grammar (PEG). This grammar
+  # supports regular expressions, and a syntax that more closely resembles
+  # other grammar languages.
+  class Peggy < Builder
+
+    def initialize
+      super
+      build
+    end
+    
+  private
+
+    # Build the parser using a builder.
+    def build
+      self.ignore_productions = [:spacing]
+      spacing{many{one{comment; eol; space}}}
+      comment{each{lit /#[^\r\n]*/; eol}}
+      eol{lit /\r\n|[\r\n]/}
+      space{lit /[ \t]/}
+      grammar{each{some{production}; eof}}
+      production{each{identifier; equal; alternates}}
+      identifier{lit /[a-zA-Z_]\w*/}
+      equal{lit ':', '='}
+      alternates{each{sequence; many{each{lit '|'; sequence}}}}
+      sequence{some{modified}}
+      modified{each{opt{prefix}; primary; opt{suffix}}}
+      primary{one{group; literal; reference}}
+      prefix{one{positive; negative}}
+      positive{lit '&'}
+      negative{lit '!'}
+      suffix{one{oneOrMore; anyNumber; optional}}
+      oneOrMore{lit '+'}
+      anyNumber{lit '*'}
+      optional{lit '?'}
+      group{each{lit '('; alternates; lit ')'}}
+      literal{one{string; regexp}}
+      reference{each{identifier; neg{equal}}}
+      string{one{lit /"(\\"|[^"\r\n])*"/; lit /'(\\'|[^'\r\n])*'/}}
+      regexp{lit /\/(\\\/|[^\/\r\n])*\//}
+    end
+
+  end
+    
+end #Peggy

data/test/test_builder.rb

@@ -0,0 +1,72 @@
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'builder')
+
+class ABCBuilder < Peggy::Builder
+
+  def build
+    step1 {
+      each {
+        lit 'a'
+        many {lit 'b'}
+        lit 'c'
+      }
+    }
+  end
+
+end
+
+class BasicEnglishBuilder < Peggy::Builder
+
+  def build
+    # self.debug_flag = true
+    self.ignore_productions = [:spacing]
+    spacing{lit /\s+/}
+    paragraph{each{some{sentence}; eof}}
+    sentence{each{
+      opt{noun_phrase}
+      verb
+      opt{noun_phrase}
+      opt{prep_phrase}
+      punctuation
+    }}
+    adjective{lit 'red', 'hot', 'picky'}
+    noun{lit 'this', 'you', 'me', 'it', 'test'}
+    noun_phrase{each{opt{particle}; opt{adjective}; noun}}
+    particle{lit 'a', 'the'}
+    preposition{lit 'for', 'about', 'to', 'from'}
+    prep_phrase{each{opt{preposition}; noun}}
+    punctuation{lit '.', '?', '!'}
+    verb{lit 'can', 'is', 'tell'}
+  end
+  
+  # Force source to lower case
+  def source_text= value
+    super value.downcase
+  end
+end
+
+class TestBuilder < Test::Unit::TestCase
+
+  def test_sequence
+    seq = Peggy::Sequence.new
+    seq << Peggy::Literal.new('a') << Peggy::Literal.new('b')
+  end
+  
+  def test_productions
+    @builder = ABCBuilder.new
+    @builder.build
+    assert_not_nil @builder.productions
+    assert_equal 1, @builder.productions.size
+    assert_equal :step1, @builder[:step1].name
+  end
+  
+  def test_basic_english
+    @builder = BasicEnglishBuilder.new
+    @builder.build
+    # @builder.debug_flag = true
+    @builder.source_text = 'This is a test. Tell me about it.'
+    result = @builder.parse?(:paragraph)
+    # pp @builder.parse_results
+    assert_equal @builder.source_text.size, result
+  end
+end

data/test/test_parser.rb

@@ -0,0 +1,119 @@
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'parser')
+
+class BasicEnglishParser < Peggy::Parser
+
+  def initialize
+    self.ignore_productions = [:spacing] # productions to ignore
+    # self.debug_flag = true # be verbose during parse
+  end
+
+  # Force source to lower case
+  def source_text= value
+    super value.downcase
+  end
+  
+  # Called and basically ignored by the parser before each match.
+  def spacing index
+    literal? /^\s+/, index # match whitespace
+  end
+
+  def paragraph index
+    while index
+      index = match? :sentence, index # don't call productions directly
+      if index
+        found = match? :eof, index # don't call eof directly either
+        return found if found
+      end
+    end
+    index
+  end
+  
+  def sentence index
+    # notice how I don't have to check index after each test
+    index = allow? :noun_phrase, index
+    # puts found
+    index = match? :verb, index
+    # puts found
+    index = allow? :noun_phrase, index
+    index = allow? :prep_phrase, index
+    match? :punctuation, index
+  end
+  
+  def adjective index
+    word index, ['red', 'hot', 'picky']
+  end
+
+  def noun index
+    word index, ['this', 'you', 'me', 'it', 'test']
+  end
+  
+  def noun_phrase index
+    index = allow? :particle, index
+    index = allow? :adjective, index
+    match? :noun, index
+  end
+  
+  def particle index
+    word index, ['a', 'the']
+  end
+  
+  def preposition index
+    word index, ['for', 'about', 'to', 'from']
+  end
+  
+  def prep_phrase index
+    index = allow? :preposition, index
+    match? :noun, index
+  end
+  
+  def punctuation index
+    word index, ['.', '?', '!']
+  end
+  
+  def verb index
+    word index, ['can', 'is', 'tell']
+  end
+  
+  def word index, list
+    list.each do |vb|
+      l2 = string? vb, index
+      return l2 if l2
+    end
+    Peggy::NO_MATCH
+  end
+end
+
+class BadParser < Peggy::Parser
+
+  def a index
+    match? :b, index
+  end
+  
+  def b index
+    match? :c, index
+  end
+
+  def c index
+    match? :a, index
+  end
+end
+
+class TestParser < Test::Unit::TestCase
+
+  def test_parse
+    @parser = BasicEnglishParser.new
+    @parser.source_text = 'This is a test. Tell me about it.'
+    result = @parser.parse? :paragraph
+    assert_equal @parser.source_text.size, result
+  end
+
+  def test_recursion
+  end
+  
+  def test_infinite_recursion
+    @parser = BadParser.new
+    assert_raise(RuntimeError) {@parser.parse?(:a, 'ab')}
+  end
+
+end

data/test/test_peg.rb

@@ -0,0 +1,54 @@
+require 'test/unit'
+require File.join(File.dirname(__FILE__), '..', 'lib', 'peg')
+require 'pp'
+
+class TestPeggy < Test::Unit::TestCase
+
+  def test_peg_grammar
+    @peg = Peggy::PEG.new
+      # note I have to double all backslashes because Ruby eats the first of each pair
+    @peg.source_text = <<-endg
+      # Hierarchical syntax
+      Grammar <- Spacing Definition+ EndOfFile
+      Definition <- Identifier LEFTARROW Expression
+      Expression <- Sequence (SLASH Sequence)*
+      Sequence <- Prefix*
+      Prefix <- (AND / NOT)? Suffix
+      Suffix <- Primary (QUESTION / STAR / PLUS)?
+      Primary <- Identifier !LEFTARROW
+        / OPEN Expression CLOSE
+        / Literal / Class / DOT
+      # Lexical syntax
+      Identifier <- IdentStart IdentCont* Spacing
+      IdentStart <- [a-zA-Z_]
+      IdentCont <- IdentStart / [0-9]
+      Literal <- ['] (!['] Char)* ['] Spacing
+        / ["] (!["] Char)* ["] Spacing
+      Class <- '[' (!']' Range)* ']' Spacing
+      Range <- Char '-' Char / Char
+      Char <- '\\\\' [nrt'"\\[\\]\\\\]
+        / '\\\\' [0-2][0-7][0-7]
+        / '\\\\' [0-7][0-7]?
+        / !'\\\\' .
+      LEFTARROW <- '<-' Spacing
+      SLASH <- '/' Spacing
+      AND <- '&' Spacing
+      NOT <- '!' Spacing
+      QUESTION <- '?' Spacing
+      STAR <- '*' Spacing
+      PLUS <- '+' Spacing
+      OPEN <- '(' Spacing
+      CLOSE <- ')' Spacing
+      DOT <- '.' Spacing
+      Spacing <- (Space / Comment)*
+      Comment <- '#' (!EndOfLine .)* EndOfLine
+      Space <- ' ' / '\\t' / EndOfLine
+      EndOfLine <- '\\r\\n' / '\\n' / '\\r'
+      EndOfFile <- !.
+    endg
+    #@peg.debug_flag = true
+    result = @peg.parse? :grammar
+    #pp @peg.parse_results
+    assert_equal @peg.source_text.length, result
+  end
+end

data/test/test_peggy.rb

@@ -0,0 +1,66 @@
+require 'test/unit'
+require 'rubygems'
+require 'peggy'
+#require File.join(File.dirname(__FILE__), '..', 'lib', 'peggy')
+require 'pp'
+
+class TestPeggy < Test::Unit::TestCase
+
+  def test_expression
+    @peggy = Peggy::Peggy.new
+      # note I have to double all backslashes because Ruby eats the first of each pair
+    @peggy.source_text = <<-ENDG
+      expression: primary (operator primary)*
+      primary
+            : literal
+            | group
+      literal: number | string
+      group: '(' expression ')'
+      operator: /[*+\\/-^%]/
+      number: int | float
+      string: /"(\\\\"|[^"])*"/
+    ENDG
+    # @peggy.debug_flag = true
+    result = @peggy.parse? :grammar
+    # pp @peggy.parse_results
+    assert_equal @peggy.source_text.length, result
+  end
+
+  def test_peggy_grammar
+    @peggy = Peggy::Peggy.new
+    @peggy.ignore_productions = [:spacing]
+      # note I have to double all backslashes because Ruby eats the first of each pair
+    @peggy.source_text = <<-endg
+      # ignored productions
+      spacing: (comment | eol | / \\t/)*
+      comment: /#[^\\r\\n]*/ eol
+      eol: /\\r\\n|[\\r\\n]/
+      # Starting production
+      grammar: production+ eof
+      production: identifier equal alternates
+      identifier: /[a-zA-Z_]\\w*/
+      equal: /[:=]/
+      alternates: sequence ('|' sequence)*
+      sequence: modified+
+      modified: prefix? primary suffix?
+      primary: group | literal | reference
+      prefix: positive | negative
+      positive: '&'
+      negative: '!'
+      suffix: oneOrMore | anyNumber | optional
+      oneOrMore: '+'
+      anyNumber: '*'
+      optional: '?'
+      group: '(' alternates ')'
+      literal: string | regexp
+      reference: identifier !equal
+      string: /"(\\\\"|[^"\\r\\n])*"/ | /'(\\\\'|[^'\\r\\n])*'/
+      regexp: /\\/(\\\\\\/|[^\\/\\r\\n])*\\//
+    endg
+    # @peggy.debug_flag = true
+    result = @peggy.parse? :grammar
+    # pp @peggy.parse_results
+    assert_equal @peggy.source_text.length, result
+  end
+
+end

data/test/tests.rb

@@ -0,0 +1,6 @@
+require 'test/unit'
+require 'rubygems'
+require 'test/test_parser'
+require 'test/test_builder'
+require 'test/test_peg'
+require 'test/test_peggy'

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.0
+specification_version: 1
+name: peggy
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-10-05 00:00:00 -06:00
+summary: A packrat parsing engine and PEG grammar, along with several other grammars.
+require_paths: 
+- lib
+email: troyhen@gmail.com
+homepage: http://rubyforge.org/projects/peggy/
+rubyforge_project: 
+description: This is a parsing library and language specifier. It uses packrat parsing, as opposed to LL(k) or LR(k) parsing. Packrat parsing uses memoization in a recursive decent parser. By storing the production results from each significant point it speeds up the parse. PEG is a formalized grammar specification optimized for packrat parsing. Peggy also allows user to specfy their grammar in pure Ruby as methods or using a Builder. And the default Peggy grammar is a  varitaion on PEG, with support for full regular expressions and for simplifed grammars which automatically ignore a set of productions.
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Troy Heninger
+files: 
+- lib/ast.rb
+- lib/builder.rb
+- lib/Copy of builder.rb
+- lib/Copy of peg.rb
+- lib/parser.rb
+- lib/peg.rb
+- lib/peggy.rb
+- test/tests.rb
+- test/test_builder.rb
+- test/test_parser.rb
+- test/test_peg.rb
+- test/test_peggy.rb
+test_files: 
+- test/tests.rb
+- test/test_builder.rb
+- test/test_parser.rb
+- test/test_peg.rb
+- test/test_peggy.rb
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+