rubypeg 0.0.2

data/lib/rubypeg.rb

@@ -0,0 +1,313 @@
+# This file contains all the elements thare are required
+# at runtime by a RubyPeg parser.
+#
+# You can either distribute it in your source code
+# or include the rubypeg gem as a dependency for 
+# your source
+
+# By default all non terminals that 
+# are returned by RubyPeg#parse are Arrays 
+# that have been extended with the NonTerminalNode
+# module
+#
+# If we consider this example:
+#   class BasketPeg < RubyPeg
+#     def root
+#       node :basket do
+#         one_or_more { items }
+#       end
+#     end
+#     
+#     def items
+#       node :item do
+#         number && optional_space && fruit && optional_space
+#       end
+#     end
+#     
+#     def number
+#       terminal(/\d+/)
+#     end
+#   
+#     def fruit
+#       node :fruit do
+#         (terminal("apple") || terminal("pear")) && ignore{ optional{ terminal("s") } }
+#       end
+#     end
+#     
+#     def optional_space
+#       ignore{ optional{ terminal(" ") }}
+#     end
+#   end
+# Then
+#   BasketPeg.parse("1 apple 2 apples 3 pears").should be_kind_of(NonTerminalNode)
+#
+# This is an array of children of this non terminal.
+# The children may be other non-terminals or terminals
+# The array will be empty if there are no children.
+#
+#   basket = BasketPeg.parse("1 apple 2 apples 3 pears")
+#   basket.class.should == Array
+#   basket.size.should == 3
+#   basket.first.should be_kind_of(NonTerminalNode)
+#   basket.first.type.should == :item
+#   basket.first.class.should == Array
+#   basket.first.size.should == 2
+#   basket.first.first.should be_kind_of(TerminalNode)
+#   basket.first.first.should == "1"
+#   basket.first.last.should be_kind_of(NonTerminalNode)
+#   basket.first.last.type == :fruit
+#   basket.first.last.class.should == Array
+#   basket.first.last.size.should == 1
+#   basket.first.last.first.should be_kind_of(TerminalNode)
+#   basket.first.last.first.should == "apple"
+module NonTerminalNode
+  
+  # Contains the argument given to RubyPeg#node
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").type.should == :basket
+  attr_accessor :type
+
+  
+  # This is a quick way of carrying out the visitor pattern on the parsed structure.
+  # 
+  # If no visitor is supplied then a nested array of child nodes is returned, with terminals turned into strings:
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").build.should == [["1", "apple"], ["2", "apple"], ["3", "pear"]]
+  # 
+  # If a visitor is supplied, then each non terminal node checks if there is a method on the visitor
+  # with a name the same as the non terminal's type. If there is, then the method is called with the
+  # children of the non terminal as arguments. If there isn't, then the build methods on the children
+  # of this node ar recursively called.
+  # E.g.,:
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").build.should == [["1", "apple"], ["2", "apple"], ["3", "pear"]]
+  #   class BasketPegBuilderExample
+  #     attr_accessor :total
+  #     
+  #     def initialize
+  #       @total = 0
+  #     end
+  #     
+  #     def item(number,kind)
+  #       @total = @total + (number.to_f * kind.build(self).to_f)
+  #     end
+  #     
+  #     def fruit(kind_of_fruit)
+  #       case kind_of_fruit
+  #       when "apple"; 3.0
+  #       when "pear"; 1.0
+  #       else  10.0
+  #       end
+  #     end
+  #   end
+  #   counter = BasketPegBuilderExample.new
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").build(counter)
+  #   counter.total.should == 12.0
+  def visit(builder = nil)
+    return builder.send(type,*self) if builder.respond_to?(type)
+    return self.first.visit(builder) if self.size == 1
+    self.map { |c| c.visit(builder) }
+  end
+  
+  # Returns the node network as an abstract syntax tree
+  #
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").to_ast.should == [:basket, [:item, "1", [:fruit, "apple"]], [:item, "2", [:fruit, "apple"]], [:item, "3", [:fruit, "pear"]]]
+  # Note that the items wrapped in ignore {} in the parser, shuch as the spaces and the optional 's' in apples and pears do not appear.
+  def to_ast
+    [type,*self.map(&:to_ast)]
+  end
+  
+  # Lists the non-terminal node and its children. Same content as #to_ast but in string form.
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").inspect.should == '[:basket, [:item, "1", [:fruit, "apple"]], [:item, "2", [:fruit, "apple"]], [:item, "3", [:fruit, "pear"]]]'
+  def inspect; to_ast.inspect end
+  
+  # Returns the result of calling to_s on each of its children. By default, TerminalNode#to_s returns its text value, so:
+  #   BasketPeg.parse("1 apple 2 apples 3 pears").to_s.should == "1apple2apple3pear"
+  # Note that the items wrapped in ignore {} in the parser, shuch as the spaces and the optional 's' in apples and pears do not appear.
+  def to_s; self.map(&:to_s).join end
+end
+
+module TerminalNode
+  def visit(builder)
+    self
+  end
+  
+  def to_ast
+    self
+  end
+end
+
+class RubyPeg
+  
+  # See #parse
+  def self.parse(text_to_parse)
+    self.new.parse(text_to_parse)
+  end
+  
+  def self.parse_and_dump(text_to_parse, dump_positive_matches_only = false)
+    e = new
+    r = e.parse(text_to_parse)
+    e.pretty_print_cache(dump_positive_matches_only)
+    r
+  end
+  
+  attr_accessor :index, :text_to_parse, :cache, :sequences
+  
+  def parse(text_to_parse)
+    self.index = 0
+    self.text_to_parse = text_to_parse
+    self.cache = {}
+    self.sequences = [[]]
+    root
+  end
+  
+  def root
+    terminal(/.*/m)
+  end
+    
+  def ignore(&block)
+    result = sequence(&block)
+    return :ignore if result
+    nil
+  end
+  
+  def any_character
+    terminal /./
+  end
+  
+  def optional
+    return yield || :ignore
+  end
+  
+  def one_or_more
+    results = []
+    while result = yield
+      results << result 
+    end
+    return nil if results.empty?
+    results
+  end
+  
+  def any_number_of
+    results = []
+    while result = yield
+      results << result 
+    end
+    results
+  end
+  
+  def sequence
+    start_index = self.index
+    self.sequences.push([])
+    if yield
+      results = self.sequences.pop
+      results.delete_if {|r| r == :ignore }
+      return results
+    else
+      self.sequences.pop
+      self.index = start_index
+      return nil
+    end
+  end
+  
+  def followed_by(&block)
+    start_index = self.index
+    result = sequence(&block)
+    self.index = start_index
+    return :ignore if result
+    return nil
+  end
+  
+  def not_followed_by(&block)
+    followed_by(&block) ? nil : :ignore
+  end
+  
+  def terminal(t)
+    return put_in_sequence(cached(t)) if cached?(t)
+    put_in_sequence(cache(t,self.index,uncached_terminal(t)))
+  end
+          
+  def node(t,&block)
+    return put_in_sequence(cached(t)) if cached?(t)
+    put_in_sequence(cache(t,self.index,uncached_node(t,&block)))
+  end
+      
+  def pretty_print_cache(only_if_match = false)
+    (0...text_to_parse.size).each do |i|
+      print "#{text_to_parse[i].inspect[1...-1]}\t#{i}\t"
+      @cache.each do |name,indexes|
+        result = indexes[i]
+        next unless result
+        if only_if_match
+          print "[#{name.inspect},#{result.first.inspect}] " if result.first
+        else
+          print "[#{name.inspect},#{result.first.inspect}] " 
+        end
+      end
+      print "\n"
+    end
+  end
+  
+  private
+
+  def uncached_terminal(t)
+    return uncached_terminal_regexp(t) if t.is_a? Regexp
+    uncached_terminal_string(t.to_s)
+  end
+  
+  def uncached_terminal_regexp(t)
+    return nil unless self.index == text_to_parse.index(t,self.index)
+    match = Regexp.last_match
+    self.index = match.end(0)
+    create_terminal_node match[0]
+  end
+  
+  def uncached_terminal_string(t)
+    return nil unless self.index == text_to_parse.index(t,self.index)
+    self.index = self.index + t.size
+    create_terminal_node t
+  end
+  
+  def create_terminal_node(text)
+    text.extend(TerminalNode)
+  end
+  
+  def uncached_node(type,&block)
+    start_index = self.index
+    results = sequence(&block)
+    return create_non_terminal_node(type,results) if results
+    self.index = start_index
+    return nil
+  end
+  
+  def create_non_terminal_node(type,children_array)
+    children_array.extend(NonTerminalNode)
+    children_array.type = type
+    children_array
+  end
+  
+  def put_in_sequence(result)
+    self.sequences.last.push(result) if result
+    result
+  end
+    
+  def cached?(name)
+    return false unless @cache.has_key?(name)
+    return false unless @cache[name].has_key?(self.index)
+    true
+  end
+  
+  def cached(name)
+    r = @cache[name][self.index]
+    self.index = r.last
+    r.first
+  end
+  
+  def cache(name,i,result)
+    if @cache.has_key?(name)
+      @cache[name][i] = [result,self.index]
+    else
+      @cache[name] = {i => [result,self.index]}
+    end
+    result
+  end
+  
+  
+end

data/lib/textpeg.rb

@@ -0,0 +1,159 @@
+require 'rubypeg'
+
+class TextPeg < RubyPeg
+  
+  def root
+    text_peg
+  end
+  
+  def text_peg
+    node :text_peg do
+      any_number_of { (spacing && (_node || definition)) }
+    end
+  end
+  
+  def _node
+    node :node do
+      identifier && assigns && expression && end_of_line
+    end
+  end
+  
+  def definition
+    node :definition do
+      identifier && equals && expression && end_of_line
+    end
+  end
+  
+  def identifier
+    node :identifier do
+      terminal(/[a-zA-Z_][a-zA-Z0-9_]*/) && spacing
+    end
+  end
+  
+  def assigns
+    ignore { terminal(":=") } && spacing
+  end
+  
+  def equals
+    ignore { terminal("=") } && spacing
+  end
+  
+  def expression
+    alternatives || _sequence
+  end
+  
+  def _sequence
+    node :sequence do
+      one_or_more { (elements && spacing) }
+    end
+  end
+  
+  def alternatives
+    node :alternatives do
+      elements && one_or_more { (divider && elements) }
+    end
+  end
+  
+  def divider
+    ignore { terminal("|") } && spacing
+  end
+  
+  def elements
+    prefixed || suffixed || element
+  end
+  
+  def prefixed
+    ignored || _not_followed_by || _followed_by
+  end
+  
+  def suffixed
+    _optional || _any_number_of || _one_or_more
+  end
+  
+  def _not_followed_by
+    node :not_followed_by do
+      ignore { terminal("!") } && element
+    end
+  end
+  
+  def _followed_by
+    node :followed_by do
+      ignore { terminal("&") } && element
+    end
+  end
+  
+  def ignored
+    node :ignored do
+      ignore { terminal("`") } && element
+    end
+  end
+  
+  def _optional
+    node :optional do
+      element && ignore { terminal("?") }
+    end
+  end
+  
+  def _any_number_of
+    node :any_number_of do
+      element && ignore { terminal("*") }
+    end
+  end
+  
+  def _one_or_more
+    node :one_or_more do
+      element && ignore { terminal("+") }
+    end
+  end
+  
+  def element
+    bracketed_expression || identifier || terminal_string || terminal_regexp || terminal_character_range || _any_character
+  end
+  
+  def bracketed_expression
+    node :bracketed_expression do
+      ignore { terminal("(") } && spacing && expression && ignore { terminal(")") } && spacing
+    end
+  end
+  
+  def terminal_string
+    node :terminal_string do
+      single_quoted_string || double_quoted_string
+    end
+  end
+  
+  def double_quoted_string
+    ignore { terminal("\"") } && terminal(/[^"]*/) && ignore { terminal("\"") } && spacing
+  end
+  
+  def single_quoted_string
+    ignore { terminal("'") } && terminal(/[^']*/) && ignore { terminal("'") } && spacing
+  end
+  
+  def terminal_character_range
+    node :terminal_character_range do
+      terminal(/\[[a-zA-Z\-0-9]*\]/) && spacing
+    end
+  end
+  
+  def terminal_regexp
+    node :terminal_regexp do
+      ignore { terminal("/") } && terminal(/(\\\/|[^\x2f])*/) && ignore { terminal("/") } && spacing
+    end
+  end
+  
+  def _any_character
+    node :any_character do
+      ignore { terminal(".") } && spacing
+    end
+  end
+  
+  def end_of_line
+    ignore { terminal(/[\n\r]+|\z/) }
+  end
+  
+  def spacing
+    ignore { terminal(/[ \t]*/) }
+  end
+  
+end

data/lib/textpeg.txt

@@ -0,0 +1,29 @@
+text_peg                        := (spacing (node | definition))*
+node                            := identifier assigns expression end_of_line
+definition                      := identifier equals expression end_of_line
+identifier                      := /[a-zA-Z_][a-zA-Z0-9_]*/ spacing
+assigns                          = `":=" spacing
+equals                           = `"=" spacing
+expression                       = alternatives | sequence
+sequence                        := (elements spacing)+
+alternatives                    := elements (divider elements)+
+divider                          = `"|" spacing
+elements                         = prefixed | suffixed | element
+prefixed                         = ignored | not_followed_by | followed_by | 
+suffixed                         = optional | any_number_of | one_or_more
+not_followed_by                 := `"!" element
+followed_by                     := `"&" element
+ignored                         := `"`" element
+optional                        := element `"?"
+any_number_of                   := element `"*"
+one_or_more                     := element `"+"
+element                          = bracketed_expression | identifier | terminal_string | terminal_regexp | terminal_character_range | any_character
+bracketed_expression            := `"(" spacing expression `")" spacing
+terminal_string                 := single_quoted_string | double_quoted_string
+double_quoted_string             = `'"' /[^"]*/ `'"' spacing
+single_quoted_string             = `"'" /[^']*/ `"'" spacing
+terminal_character_range        := /\[[a-zA-Z\-0-9]*\]/ spacing
+terminal_regexp                 := `'/' /(\\\/|[^\x2f])*/ `'/' spacing
+any_character                   := `'.' spacing
+end_of_line                      = `/[\n\r]+|\z/
+spacing                          = `/[ \t]*/

data/lib/textpeg2rubypeg.rb

@@ -0,0 +1,182 @@
+require 'text_peg'
+
+class String
+  
+  def to_class_name
+    # Taken from ActiveSupport inflector
+    self.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+  end
+  
+  def to_method_name
+    # Taken from ActiveSupport inflector
+    self.gsub(/::/, '/').
+       gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+       gsub(/([a-z\d])([A-Z])/,'\1_\2').
+       tr("-", "_").
+       downcase
+  end
+  
+end
+
+class TextPeg2RubyPeg
+  
+
+  def TextPeg2RubyPeg.parse_to_ruby(text_peg)
+    TextPeg.parse(text_peg).visit(TextPeg2RubyPeg.new)
+  end
+
+  def TextPeg2RubyPeg.parse_to_loaded_class(text_peg)
+    builder = TextPeg2RubyPeg.new
+    ruby =  TextPeg.parse(text_peg).visit(builder)
+    Kernel.eval(ruby)
+    Kernel.eval(builder.class_name)
+  end
+
+  def TextPeg2RubyPeg.parse_file_to_loaded_class(filename)
+    parse_to_loaded_class IO.readlines(filename).join
+  end
+  
+  attr_accessor :ruby,:tabs,:class_name #:nodoc:
+  
+  RESERVED_WORDS = %w{index text_to_parse cache sequences parse ignore any_character optional one_or_more any_number_of sequence followed_by not_followed_by uncached_terminal uncached_terminal_regexp uncached_terminal_string create_terminal_node create_non_terminal_node uncached_node terminal node put_in_sequence cached? cached cache pretty_print_cache}   #:nodoc:
+  
+  def identifier(name) #:nodoc:
+    return name.to_s unless RESERVED_WORDS.include?(name.to_s)
+    $stderr.puts "Identifier #{name} clashes with a reserved word in the parser, replacing with _#{name}"
+    "_#{name}"
+  end
+  
+  def text_peg(*definitions) #:nodoc:
+    self.ruby = []
+    self.tabs = 0
+    definitions.map { |d| d.visit(self) }
+    close_class
+    to_ruby
+  end
+  
+  def definition(identifier,expression) #:nodoc:
+    non_clashing_name = identifier.visit(self)
+    unless class_name
+      define_class non_clashing_name
+      define_root non_clashing_name
+    end
+    line "def #{non_clashing_name.to_method_name}"
+    indent
+    line expression.visit(self)
+    outdent
+    line "end"
+    line
+  end
+  
+  def node(identifier,expression) #:nodoc:
+    original_name = identifier.to_s
+    non_clashing_name = identifier.visit(self)
+    unless class_name
+      define_class non_clashing_name
+      define_root non_clashing_name
+    end
+    line "def #{non_clashing_name.to_method_name}"
+    indent
+    line "node :#{original_name.to_method_name} do"
+    indent
+    line expression.visit(self)
+    outdent
+    line "end"
+    outdent
+    line "end"
+    line
+  end
+  
+  def define_class(name) #:nodoc:
+    self.class_name = name.to_class_name
+    line "require 'rubypeg'"
+    line ""
+    line "class #{class_name} < RubyPeg"
+    indent
+    line
+    @first_definition  = false
+  end
+  
+  def define_root(name) #:nodoc:
+    line "def root"
+    indent
+    line name.to_method_name
+    outdent
+    line "end"
+    line
+  end
+  
+  def not_followed_by(element) #:nodoc:
+    "not_followed_by { #{element.visit(self)} }"
+  end
+  
+  def followed_by(element) #:nodoc:
+    "followed_by { #{element.visit(self)} }"
+  end
+  
+  def ignored(element) #:nodoc:
+    "ignore { #{element.visit(self)} }"
+  end
+    
+  def optional(element) #:nodoc:
+    "optional { #{element.visit(self)} }"
+  end
+  
+  def one_or_more(element) #:nodoc:
+    "one_or_more { #{element.visit(self)} }"
+  end
+  
+  def any_number_of(element) #:nodoc:
+    "any_number_of { #{element.visit(self)} }"
+  end
+  
+  def sequence(*elements) #:nodoc:
+    elements.map { |e| e.visit(self) }.join(" && ")
+  end
+  
+  def alternatives(*elements) #:nodoc:
+    elements.map { |e| e.visit(self) }.join(" || ")    
+  end
+  
+  def bracketed_expression(expression) #:nodoc:
+    "(#{expression.visit(self)})"
+  end
+  
+  def terminal_string(string) #:nodoc:
+    %Q{terminal(#{string.visit(self).inspect})}
+  end
+
+  def terminal_regexp(regexp) #:nodoc:
+    "terminal(/#{regexp.visit(self)}/)"
+  end
+  
+  def terminal_character_range(regexp) #:nodoc:
+    "terminal(/#{regexp.visit(self)}/)"
+  end
+  
+  def any_character #:nodoc:
+    "any_character"
+  end
+  
+  def close_class #:nodoc:
+    outdent
+    line "end\n"
+  end
+  
+  def line(string = "") #:nodoc:
+    ruby << "#{"  "*tabs}#{string}"
+  end
+  
+  def indent #:nodoc:
+    self.tabs = tabs + 1
+  end
+  
+  def outdent #:nodoc:
+    self.tabs = tabs - 1
+  end
+  
+  def to_ruby #:nodoc:
+    ruby.join("\n")
+  end
+  
+end

data/spec/any_character_spec.rb

@@ -0,0 +1,22 @@
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+require 'rubypeg'
+
+class AnyCharacter < RubyPeg
+  def root
+    node :root do 
+      any_character
+    end
+  end  
+end
+
+describe AnyCharacter do
+  
+  it "matches one of any character" do
+    AnyCharacter.parse("abcd").to_ast.should == [:root,'a']
+  end
+  
+  it "doesn't match no character" do
+    AnyCharacter.parse("").should == nil
+  end
+    
+end