peggy 0.1.0

data/lib/ast.rb

@@ -0,0 +1,86 @@
+module Peggy
+  class Node
+    attr_accessor :name, :first, :next, :parent, :from, :to
+    
+    def initialize name
+      self.name = name
+    end
+    
+    def << child
+      child.parent = self
+      if first
+        last.next = child
+      else
+        first = child
+      end
+    end
+       
+    def each
+      child = first
+      while child
+        yield child
+        child = child.next
+      end
+    end
+    
+    def format tabs
+      result = "#{tabs}#{self}\n"
+      tabs << '  '
+      node = first
+      while node
+        "#{tabs}#{node.format tabs}"
+      end
+      tabs = tabs[0..-3]
+    end
+    
+    def last
+      node = first
+      while (n2 = node.next)
+        node = n2
+      end
+    end
+    
+    def to_s source=nil
+      source ? source[from...to] : "#{name}[#{from}...#{to}]"
+    end
+  end
+  
+  class AST
+  
+    attr_reader :root
+    
+    def initialize
+      @root = Node.new
+      build result, index, @root
+    end
+
+    def self::build parser, index = 0
+      AST.new.build_one results, index, root
+    end
+    
+    private
+    
+    def build_one results, index, parent
+      row = results[index]
+      row = results[index += 1] until row
+      top = parent
+      results[:found_order].reverse_each do |name|
+        node = Node.new name
+        node.from = index
+        node.to = row[name]
+        build_rest results, node
+        top << node
+        node = top
+      end
+      
+    end
+    
+    def build_rest results, previous
+      build_one results, previous.to, previous.parent
+    end
+    
+    def to_s
+      @root.format ''
+    end
+  end
+end # Peggy

data/lib/builder.rb

@@ -0,0 +1,359 @@
+require 'rubygems'
+require 'parser'
+# require File.join(File.dirname(__FILE__), 'parser')
+
+module Peggy
+
+  # Base syntax element class.
+  class Element
+    # Create an element.
+    def self::build *args
+      new *args
+    end
+    
+    # Test to see if there is a match of this element at the current index.
+    # Return's the index following if match is found, or NO_MATCH if not
+    def match parser, index
+      raise "Must override match"
+    end
+    
+    def report index
+      # puts "#{to_s} #{index}"
+      index
+    end
+  end
+
+  # An element that matches a sequence of elements.  All must match for the sequence to match.
+  class Sequence < Element
+    # Add a child element.
+    def add element
+      @list = [] unless @list
+      @list << element
+    end
+
+    # Synonym for add(element)
+    alias :'<<' :add
+    
+    # Reference a child by index.
+    def [] index
+      @list[index]
+    end
+    
+    # Child iterator.
+    def each &blk
+      @list.each &blk
+    end
+    
+    # Match each child in sequence. If any fail this returns NO_MATCH. If all succeed this 
+    # returns the end index of the last.
+    def match parser, index
+      raise "no children added to sequence" unless @list
+      each do |element|
+        index = element.match parser, index
+        return NO_MATCH unless index
+      end
+      report index
+    end
+  end
+  
+  # An element which matches any one of its children. The children are tested in order. The first
+  # to match wins.
+  class Alternatives < Sequence
+    # Match any one of the children. The children are tried in order. The first to match wins.
+    # The result is the end index of the first matching child. If none match this returns NO_MATCH.
+    def match parser, index
+      raise "no children added to alternate" unless @list
+      each do |element|
+        found = element.match parser, index
+        return report(found) if found
+      end
+      report NO_MATCH
+    end
+  end
+  
+  # An element which tries its single child multiple times. It is greedy, meaning it will continue 
+  # to match as long as possible, unless the range specifies a maximum number of matches.
+  class Multiple < Element
+    # A big number
+    MANY = 32767
+    # The minimum and maximum number of tries
+    attr_accessor :range
+    # The single child
+    attr_accessor :child
+
+    # Init the range
+    def initialize range
+      @range = range
+    end
+    
+    # synonym for child=(element)
+    alias :'<<' :'child='
+    
+    # Matches the child multiple times. The range specifies the least and most number of matches.
+    # If the number of matches is less than the minimim of the range then NO_MATCH is returned.
+    # If equal or more than the minimim then the end index of the last match is returned.
+    def match parser, index
+      raise "multiple element child not set" unless child
+      raise "multiple element range not set" unless range
+      count = 0
+      while count < range.last
+        found = child.match parser, index
+        break unless found
+        index = found
+        count += 1
+      end
+      report range === count ? index : NO_MATCH
+    end
+  end
+  
+  # Matcher of 0 or more times.
+  class AnyNumber < Multiple
+    def initialize
+      super 0..MANY
+    end
+  end
+  
+  # Matcher of 1 or more times.
+  class AtLeastOne < Multiple
+    def initialize
+      super 1..MANY
+    end
+  end
+  
+  # Matcher of 0 or 1 time.
+  class Optional < Multiple
+    def initialize
+      super 0..1
+    end
+  end
+  
+  # An element which tries its single child but does not advance the index if found.
+  # If not found, however, it returns NO_MATCH. Used for a positive semantic predicate.
+  class Positive < Element
+    # The single child
+    attr_accessor :child
+    
+    # synonym for child=(element)
+    alias :'<<' :'child='
+    
+    # Matches the child once. If found the original index is returned.
+    # If not found NO_MATCH is returned.
+    def match parser, index
+      raise "positive element child not set" unless child
+      found = child.match parser, index
+      found ? index : NO_MATCH
+    end
+  end
+  
+  # An element which tries its single child but does not advance the index if not found.
+  # If found, however, it returns NO_MATCH. Used for a negative semantic predicate.
+  class Negative < Positive
+    def match parser, index
+      raise "negative element child not set" unless child
+      found = child.match parser, index
+      found ? NO_MATCH : index
+    end
+  end
+  
+  # Match another production in the grammar.
+  class Reference < Element
+    # The name of the production to lookup and match.
+    attr_reader :name
+
+    # Init the name
+    def initialize name=nil
+      self.name = name
+    end
+
+    # Set the name of production to match.
+    def name= value
+      @name = value.to_sym
+    end
+
+    # Match the entire production from the parser grammar. If it matches
+    # the end index is returned. If not, NO_MATCH is returned.
+    def match parser, index
+      raise "reference name not set" unless name
+      parser.match? name, index
+    end
+    
+    def to_s
+      name
+    end
+  end
+  
+  # Matcher of a grammar production. The one and only child defines the production.
+  class Production < Reference
+    # The production definition.
+    attr_accessor :child
+
+    # Init the name and child.
+    def initialize name=nil, child=nil
+      super name
+      @child = child
+    end
+
+    # Synonym of child=(element)
+    alias :'<<' :'child='
+    
+    # Match the production one time. If it matches the end index is returned. If not,
+    # NO_MATCH is returned.
+    def match parser, index
+      raise "production name not set" unless name
+      raise "production child not set" unless child
+      report @child.match(parser, index)
+    end
+  end
+  
+  # Matcher of a literal string or regular expression.
+  class Literal < Element
+    # Value to match.
+    attr_reader :value
+    
+    # Init the value.
+    def initialize value=nil
+      @value = value
+    end
+
+    # Set the value to match.
+    def value= literal
+      # Make sure regular expressions check at the beginnig of the string
+      literal = correct_regexp literal if literal.is_a? Regexp
+      @value = literal
+    end
+    
+    # Match the literal value. If it matches the end index is returned.
+    # If no, NO_MATCH is returned.
+    def match parser, index
+      report parser.literal?(value, index)
+    end
+    
+    def to_s
+      value.inspect
+    end
+  end
+
+  # Parser builder. The built in methods create syntax elements. Any other
+  # method called on this object create references to production, or actual
+  # productions, if called at the top level.
+  # Todo: Change to a class and separate from Parser.
+  class Builder < Parser
+    # Productions to build
+    attr_reader :productions
+    # Current parent being built
+    attr_reader :parent
+    
+    def initialize
+      @building = true
+    end
+    
+    # Reference a production by its name index.
+    def [] index
+      productions[index]
+    end
+    
+    # Create a production if at the top level, or a reference to a production a 
+    # production is being built.
+    def method_missing name, *args
+      if @building
+        if @parent
+          ref = Reference.new name
+          @parent << ref
+        elsif block_given?
+          @productions = {} unless @productions
+          prod = Production.new name
+          @parent = prod
+          yield
+          @parent = nil
+          @productions[name] = prod
+        else
+          super
+        end
+      else
+        prod = @productions[name]
+        super unless prod
+# puts "matching #{name} at #{args.first}"
+        prod.match self, args.first
+      end
+    end
+
+    # Add an Alternatives element to the parent.
+    def one &blk
+      build_piece Alternatives, blk
+    end
+    # Synonym for one().
+    alias :alt :one
+  
+    def eof *args
+      if args.length == 1 then super args.first
+      else method_missing :eof, *args
+      end
+    end
+    
+    # Add an Sequence element to the parent.
+    def each &blk
+      build_piece Sequence, blk
+    end
+    # Synonym for each()
+    alias :seq :each
+  
+    # Add an Literal element to the parent.
+    def lit *values
+      if values.size == 1
+        build_piece Literal, nil, values.first
+      else
+        one{
+          for v in values
+            build_piece Literal, nil, v
+          end
+        }
+      end
+    end
+  
+    # Add an AnyNumber element to the parent.
+    def many &blk
+      build_piece AnyNumber, blk
+    end
+  
+    # Add an Optional element to the parent.
+    def opt &blk
+      build_piece Optional, blk
+    end
+  
+    # Add an AtLeastOne element to the parent.
+    def some &blk
+      build_piece AtLeastOne, blk
+    end
+
+    def neg &blk
+      build_piece Negative, blk
+    end
+
+    def pos &blk
+      build_piece Positive, blk
+    end
+
+    def parse? goal, index=0
+      @building = nil
+      super
+    end
+
+    private
+    
+    # Add an object of klass to the parent and yield to its block. If 
+    # value is specified it is passed to the klass constructor.
+    def build_piece klass, blk=nil, value=nil
+# puts "building #{klass.name} with #{value.inspect}"
+      elem = value ? klass.new(value) : klass.new
+      @parent << elem
+      if blk
+        parent = @parent
+        @parent = elem
+        blk.call
+        @parent = parent
+      end
+    end
+    
+  end # Builder
+
+end # Peggy

data/lib/parser.rb

@@ -0,0 +1,203 @@
+require 'pp'
+
+# Peggy is a packrat parsing engine. Packrat parsers memoize every production so that 
+# parses can happen in linear time. No production needs to be processed more than once for 
+# a given position of the source. See http://pdos.csail.mit.edu/~baford/packrat/ for
+# more details. 
+# 
+# Peggy also incorporates Parsing Expression Grammar (PEG) as proposed by Bryan Ford,
+# as one of several input grammars. PEG is a formalized grammar specification needing 
+# no separate lexer/scanner step. See http://pdos.csail.mit.edu/~baford/packrat/popl04/
+# 
+# As good as packrat parsers are, they have a few limitations. They cannot handle left
+# recursion of a production, meaning a production cannot reference itself as the first
+# element in a sequence. Also memoizing of production results means than memory consumption
+# increasses with the size of the source being parsed. This is not usaly a concern, execpt
+# when attempting to parse multi-megabyte source files, such as a huge XML database.
+module Peggy
+
+  # Returned when a production did not match
+  NO_MATCH = false
+  # Used to prevent infinite (left) recursions
+  IN_USE = true
+  
+  # Packrat parser class. Note all methods have a trailing exclamation (!) or question 
+  # mark (?), or have long names with underscores (_). This is because productions are 
+  # methods and we need to avoid name collisions. To use this class you must subclass 
+  # Parser and provide your productions as methods. Your productions must call match? 
+  # or one of the protected convenience routines to perform parsing. Productions must 
+  # never call another production directly, or results will not get memoized and you 
+  # will slow down your parse conciderably, and possibly risk getting into an infinite 
+  # recursion (until the stack blows its top). Note, as a conveience in writting 
+  # productions, you can call any match? function multiple times, passing each returned 
+  # index, such as in a sequence, without checking the results of each production.
+  class Parser
+    
+    # Tells parser to print intermediate results if set.
+    attr_accessor :debug_flag
+
+    # The source to parse, can be set prior to calling parse!().
+    attr_accessor :source_text
+    
+    # The results of the parse. A hash (keys of indexs) of hashes (keys of production 
+    # symbols and values of end indexes.
+    attr_reader :parse_results
+
+    # The productions to ignore.
+    attr_accessor :ignore_productions
+
+    # Return a range (or character) of the source_text.
+    def [] range
+      raise "source_text not set" if source_text.nil?
+      source_text[range]
+    end
+
+    # Envokes the parser from the beginning of the source on the given production goal.
+    # You sohuld provide the source here or you can set source_text prior to calling.
+    # If index is provided the parser will ignore characters previous to it.
+    def parse? goal, source = nil, index = 0
+      source_text = source unless source.nil?
+        # Hash of automatic hashes
+      @parse_results = Hash.new {|h1, k1| h1[k1] = {}}
+      @keys = nil
+      index = match? goal, index
+      puts pp(parse_results) if debug_flag
+      index
+    end
+
+    # Queries the parse results for a heirarchy of production matches. An array of 
+    # index ranges is returned, or an empny array if none are found. This can only be 
+    # called after parse_results have been set by a parse.
+    def query? *args
+      raise "You must first call parse!" unless parse_results
+      @keys = @parse_results.keys.sort unless @keys
+      found_list = []
+      index = 0
+      args.each do |arg|
+        index = find? arg, index
+      end
+    end
+    
+    # Try to match a production from the given index. Returns the end index if found 
+    # or start index if not found.
+    def allow? goal, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      found = match? goal, index
+      found == NO_MATCH ? index : found
+    end
+    
+    # Try to match a production from the given index then backtrack. Returns index if 
+    # found or NO_MATCH if not.
+    def check? goal, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      found = match? goal, index
+      found == NO_MATCH ? NO_MATCH : index
+    end
+    
+    # Try not to match a production from the given index then backtrack. Returns index 
+    # if not found or NO_MATCH if found.
+    def dissallow? goal, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      found = match? goal, index
+      found == NO_MATCH ? index : NO_MATCH
+    end
+    
+    # Special production that only matches the end of source_text. Note, this function
+    # does not end in (?) or (!) because it is meant be used as a normal production.
+    def eof index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      index >= source_text.length ? index : NO_MATCH
+    end
+    
+    # Match a production from the given index. Returns the end index if found or NO_MATCH
+    # if not found.
+    def match? goal, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      index = ignore? index unless @ignoring
+      goal = goal.to_sym
+      position = parse_results[index]
+      found = position.fetch(goal) do
+        position[goal] = IN_USE # used to prevent inifinite recursion in case user attemts 
+                                # a left recursion
+        if (result = send goal, index)
+          position[:found_order] = [] unless position.has_key?(:found_order)
+          position[:found_order] << goal
+        end
+        position[goal] = result
+      end
+      puts "found #{goal} at #{index}...#{found} #{source_text[index...found].inspect}" if found && debug_flag
+      raise "Parser cannot handle infinite (left) recursions. Please rewrite usage of '#{goal}'." if found == IN_USE
+      found
+    end
+
+    # Match tokens that should be ignored. Used by match?(). Returns end index if found 
+    # or start index if not found. Subclasses should override this method if they wish 
+    # to ignore other text, such as comments.
+    def ignore? index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      return index if @ignoring || ignore_productions.nil?
+      @ignoring = true
+      ignore_productions.each do |prod|
+        index = allow? prod, index
+      end
+      @ignoring = nil
+      index
+    end
+
+    # Match a literal string or regular expression from the given index. Returns 
+    # the end index if found or NO_MATCH if not found.
+    def literal? value, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      case value
+      when String
+        string? value, index
+      when Regexp
+        regexp? value, index
+      else
+        raise "Unknown literal: #{value.inspect}"
+      end
+    end
+
+    # Match a string from the given index. Returns the end index if found 
+    # or NO_MATCH if not found.
+    def string? value, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      value = value.to_s
+      index = ignore? index unless @ignoring
+      i2 = index + value.length
+# puts source_text[index...i2].inspect + ' ' + value.inspect
+      source_text[index...i2] == value ? i2 : NO_MATCH
+    end
+
+    # Match a regular expression from the given index. Returns the end index 
+    # if found or NO_MATCH if not found.
+    def regexp? value, index
+      return NO_MATCH if index == NO_MATCH # allow users to not check results of a sequence
+      value = correct_regexp! value
+      index = ignore? index unless @ignoring
+      found = value.match source_text[index..-1]
+# puts "#{value.inspect} ~= #{found[0].inspect}" if found
+      found ? found.end(0) + index : NO_MATCH
+    end
+    
+    # Make sure regular expressions match the beginning of the string, actually from 
+    # the string from the given index.
+    def correct_regexp! re
+      source = re.source
+      source[0..1] == '\\A' ? re : Regexp.new("\\A(#{source})", re.options)
+    end
+    
+    protected
+    
+    def index_results!
+      raise "You must first call parse!" unless parse_results
+      @index = new Hash {|h, k| h[k] = []}
+      parse_results.each_pair do |index, prod_map|
+        prod_map[:found_order].reverse_each
+        prod_map.each_value
+        @index[prod]
+      end
+    end
+  end # Parser
+  
+end # Peggy