babel_bridge 0.1.1 → 0.2.0

data/babel_bridge.gemspec

@@ -10,7 +10,11 @@ $gemspec = Gem::Specification.new do |s|
   s.platform = Gem::Platform::RUBY
   s.rubyforge_project = "babel-bridge"
   s.summary = "A Ruby-based parser-generator based on Parsing Expression Grammars."
-  s.description = "Babel Bridge let's you generate parsers 100% in Ruby code. It is a memoizing Parsing Expression Grammar (PEG) generator like Treetop, but it doesn't require special file-types or new syntax. Overall focus is on simplicity and usability over performance."
+  s.description = <<DESCRIPTION
+Babel Bridge is an object oriented parser generator for parsing expression grammars (PEG). 
+Generate memoizing packrat parsers 100% in Ruby code with a simple embedded DSL.
+DESCRIPTION
+  
   s.files = ["LICENSE", "README", "Rakefile", "babel_bridge.gemspec", "{test,lib,doc,examples}/**/*"].map{|p| Dir[p]}.flatten
   s.has_rdoc = false
 end

data/lib/babel_bridge.rb

@@ -1,33 +1,11 @@
 =begin
-
-See README
-
-TODO-FEATURE: :pre_delimiter option
-TODO-FEATURE: The "expecting" feature is so good I wonder if we should add the ability to automatically repair the parse!
-  This would need:
-    a) default values for regex termainals (string terminals are their own default values)
-      default values for regex should be verified to match the regex
-    b) an interactive prompter if there is more than one option
-
-TODO-IMPROVEMENT: "Expecting" should show line numbers instead of char numbers, but it should only calculated
-  on demand. This means we need a smarter formatter for our possible-error-logging.
-TODO-IMPROVEMENT: "Expecting" code lines dump should show line numbers
-
-TODO-BUG: "Expecting" doesn't do the right thing of a "dont" clause matched
-  Should say "something other than #{the don't clause}"
-  Ideally, we would continue matching and list all the possible next clauses that would allow us to continue
-
-IDEA: could use the "-" prefix operator to mean "dont":
-  -"this"
-  -:that
-  -match(:foo)
-  -many(:foo)
-
-TODO-OPTIMIZATION: add memoizing (caching / dynamic-programming) to guarantee linear time parsing
-  http://en.wikipedia.org/wiki/Parsing_expression_grammar#Implementing_parsers_from_parsing_expression_grammars
+Copyright 2011 Shane Brinkman-Davis
+See README for licence information.
+http://babel-bridge.rubyforge.org/
 =end
 
 require File.dirname(__FILE__) + "/nodes.rb"
+require File.dirname(__FILE__) + "/pattern_element.rb"
 
 class String
   def camelize
@@ -51,164 +29,7 @@ class String
 end
 
 module BabelBridge
-  VERSION = "0.1.1"
-
-  # hash which can be used declaratively
-  class PatternElementHash < Hash
-    def method_missing(method_name, *args)  #method_name is a symbol
-      return self if args.length==1 && !args[0] # if nil is provided, don't set anything
-      self[method_name]=args[0] || true # on the other hand, if no args are provided, assume true
-      self
-    end
-  end
-
-  # PatternElement provides optimized parsing for each Element of a pattern
-  # PatternElement provides all the logic for parsing:
-  #   :many
-  #   :optional
-  class PatternElement
-    attr_accessor :parser,:optional,:negative,:name,:terminal,:could_match
-    attr_accessor :match,:rule_variant
-
-    #match can be:
-    # true, Hash, Symbol, String, Regexp
-    def initialize(match,rule_variant)
-      self.rule_variant=rule_variant
-      init(match)
-
-      raise "pattern element cannot be both :dont and :optional" if negative && optional
-    end
-
-    def to_s
-      match.inspect
-    end
-
-    def parse(parent_node)
-      # run element parser
-      match=parser.call(parent_node)
-
-      # Negative patterns (PEG: !element)
-      match=match ? nil : EmptyNode.new(parent_node) if negative
-
-      # Optional patterns (PEG: element?)
-      match=EmptyNode.new(parent_node) if !match && optional
-
-      # Could-match patterns (PEG: &element)
-      match.match_length=0 if match && could_match
-
-      # return match
-      match
-    end
-
-    private
-
-    def init(match)
-      self.match=match
-      case match
-      when TrueClass then init_true
-      when Hash then      init_hash match
-      when Symbol then    init_rule match
-      when String then    init_string match
-      when Regexp then    init_regex match
-      else                raise "invalid pattern type: #{match.inspect}"
-      end
-    end
-
-    def init_rule(rule_name)
-      rule_name.to_s[/^([^?!]*)([?!])?$/]
-      rule_name=$1.to_sym
-      option=$2
-      match_rule=rule_variant.rule.parser.rules[rule_name]
-      raise "no rule for #{rule_name}" unless match_rule
-
-      self.parser =lambda {|parent_node| match_rule.parse(parent_node)}
-      self.name   = rule_name
-      case option
-      when "?"  then self.optional=true
-      when "!"  then self.negative=true
-      end
-    end
-
-    def init_hash(hash)
-      if hash[:parser]
-        self.parser=hash[:parser]
-      elsif hash[:many]
-        init hash[:many]
-        #generate parser for poly
-        delimiter_pattern_element= PatternElement.new(hash[:delimiter]||true,rule_variant)
-
-        post_delimiter_element=case hash[:post_delimiter]
-          when TrueClass then delimiter_pattern_element
-          when nil then nil
-          else PatternElement.new(hash[:post_delimiter],rule_variant)
-          end
-
-        # convert the single element parser into a poly-parser
-        single_parser=parser
-        self.parser= lambda do |parent_node|
-          last_match=single_parser.call(parent_node)
-          many_node=ManyNode.new(parent_node)
-          while last_match
-            many_node<<last_match
-
-            #match delimiter
-            delimiter_match=delimiter_pattern_element.parse(many_node)
-            break unless delimiter_match
-            many_node.delimiter_matches<<delimiter_match
-
-            #match next
-            last_match=single_parser.call(many_node)
-          end
-
-          # success only if we have at least one match
-          return nil unless many_node.length>0
-
-          # pop the post delimiter matched with delimiter_pattern_element
-          many_node.delimiter_matches.pop if many_node.length==many_node.delimiter_matches.length
-
-          # If post_delimiter is requested, many_node and delimiter_matches must be the same length
-          if post_delimiter_element
-            post_delimiter_match=post_delimiter_element.parse(many_node)
-
-            # fail if post_delimiter didn't match
-            return nil unless post_delimiter_match
-            many_node.delimiter_matches<<post_delimiter_match
-          end
-
-          many_node
-        end
-      elsif hash[:match]
-        init hash[:match]
-      else
-        raise "extended-options patterns (specified by a hash) must have either :parser=> or a :match=> set"
-      end
-
-      self.name = hash[:as] || self.name
-      self.optional ||= hash[:optional] || hash[:optionally]
-      self.could_match ||= hash[:could]
-      self.negative ||= hash[:dont]
-
-    end
-
-    # "true" parser always matches the empty string
-    def init_true
-      self.parser=lambda {|parent_node| EmptyNode.new(parent_node)}
-    end
-
-    # parser that matches exactly the string specified
-    def init_string(string)
-      self.parser=lambda {|parent_node| parent_node.src[parent_node.next,string.length]==string && TerminalNode.new(parent_node,string.length,string)}
-      self.terminal=true
-    end
-
-    # parser that matches the given regex
-    def init_regex(regex)
-      self.parser=lambda {|parent_node| offset=parent_node.next;parent_node.src.index(regex,offset)==offset && (o=$~.offset(0)) && TerminalNode.new(parent_node,o[1]-o[0],regex)}
-      self.terminal=true
-    end
-
-  end
-
+  VERSION = "0.2.0"
 
   # Each Rule has one or more RuleVariant
   # Rules attempt to match each of their Variants in order. The first one to succeed returns true and the Rule succeeds.
@@ -268,7 +89,7 @@ module BabelBridge
       self.parser=parser
 
       class_name = "#{parser.module_name}_#{name}_node".camelize
-      self.node_class = parser.const_set(class_name,Class.new(NodeNT))
+      self.node_class = parser.const_set(class_name,Class.new(NonTerminalNode))
     end
 
     def add_variant(pattern, &block)
@@ -286,8 +107,7 @@ module BabelBridge
       end
 
       variants.each do |v|
-        match=v.parse(node)
-        if match
+        if match=v.parse(node)
           node.parser.cache_match(name,match)
           return match
         end

data/lib/nodes.rb

@@ -1,3 +1,9 @@
+=begin
+Copyright 2011 Shane Brinkman-Davis
+See README for licence information.
+http://babel-bridge.rubyforge.org/
+=end
+
 module BabelBridge
 
   # this is just so we can distinguish between normal arrays and arrays of matches
@@ -8,7 +14,7 @@ module BabelBridge
   # base class for all parse-tree nodes
   class Node
     attr_accessor :src,:offset,:match_length,:parent,:parser
-    
+
     def to_s
       text
     end
@@ -81,14 +87,14 @@ module BabelBridge
     def each(&block)
       matches.each(&block)
     end
-end
+  end
 
   class RootNode < Node
   end
 
   # non-terminal node
   # subclassed automatically by parser.rule for each unique non-terminal
-  class NodeNT < Node
+  class NonTerminalNode < Node
     attr_accessor :matches,:match_names
 
     def match_names
@@ -151,22 +157,27 @@ end
       @matches_by_name=nil
     end
 
+    # defines where to forward missing methods to; override for custom behavior
+    def forward_to
+      matches[0]
+    end
+
     def method_missing(method_name, *args)  #method_name is a symbol
       unless matches_by_name.has_key? method_name
-        if matches[0]
-          puts "sending #{method_name.inspect} to #{matches[0].class}"      
-          return matches[0].send(method_name,*args)
+        if f=forward_to
+          return f.send(method_name,*args)
         end
         raise "#{self.class}: missing method #{method_name.inspect} / doesn't match named pattern element: #{matches_by_name.keys.inspect}"
       end
-      matches_by_name[method_name]
+      case ret=matches_by_name[method_name]
+      when EmptyNode then nil
+      else ret
+      end
     end
 
     # adds a match with name (optional)
     # returns self so you can chain add_match or concat methods
     def add_match(match,name=nil)
-      raise "match must be a Node (match is a #{match.class})" unless match.kind_of?(Node)
-      raise "name must be a Symbol or nil (name is a #{name.class})" if name && !name.kind_of?(Symbol)
       reset_matches_by_name
       matches<<match
       match_names<<name
@@ -192,16 +203,22 @@ end
       node_init(parent)
       self.matches=[]
       self.delimiter_matches=[]
-      self.match_length=nil   # use match_length as an override; if nil, then match_length is determined by the last node and delimiter_match
     end
 
-    def match_length; @match_length || (self.next-offset) end
+    def match_length; self.next-offset end
+
     def next
-      return offset+@match_length if @match_length
-      ret=nil
-      ret=matches[-1].next if matches[-1] && (!ret || matches[-1].next > ret)
-      ret=delimiter_matches[-1].next if delimiter_matches[-1] && (!ret || delimiter_matches[-1].next > ret)
-      ret||=parent.next
+      if m=matches[-1]
+        m_next=m.next
+        if d=delimiter_matches[-1]
+          d_next=d.next
+          m_next > d_next ? m_next : d_next
+        else
+          m_next
+        end
+      else
+        parent.next
+      end
     end
 
     def inspect_helper(list,options)
@@ -226,6 +243,11 @@ end
         ret
       end
     end
+
+    def method_missing(method_name, *args)  #method_name is a symbol
+      self.map {|match| match.send(method_name,*args)}
+    end
+
   end
 
   # used for String and Regexp PatternElements
@@ -239,7 +261,7 @@ end
     end
 
     def inspect(options={})
-      "#{text.inspect}" unless options[:simple] && text[/^\s*$/] # if simple && node only matched white-space, return nil 
+      "#{text.inspect}" unless options[:simple] && text[/^\s*$/] # if simple && node only matched white-space, return nil
     end
 
     def matches; [self]; end

data/lib/pattern_element.rb

@@ -0,0 +1,193 @@
+=begin
+Copyright 2010 Shane Brinkman-Davis
+See README for licence information.
+http://babel-bridge.rubyforge.org/
+=end
+
+module BabelBridge
+  # hash which can be used declaratively
+  class PatternElementHash < Hash
+    def method_missing(method_name, *args)  #method_name is a symbol
+      return self if args.length==1 && !args[0] # if nil is provided, don't set anything
+      self[method_name]=args[0] || true # on the other hand, if no args are provided, assume true
+      self
+    end
+  end
+
+  # PatternElement provides optimized parsing for each Element of a pattern
+  # PatternElement provides all the logic for parsing:
+  #   :many
+  #   :optional
+  class PatternElement
+    attr_accessor :parser,:optional,:negative,:name,:terminal,:could_match
+    attr_accessor :match,:rule_variant
+
+    #match can be:
+    # true, Hash, Symbol, String, Regexp
+    def initialize(match,rule_variant)
+      self.rule_variant=rule_variant
+      init(match)
+
+      raise "pattern element cannot be both :dont and :optional" if negative && optional
+    end
+
+    def to_s
+      match.inspect
+    end
+
+    # attempt to match the pattern defined in self.parser in parent_node.src starting at offset parent_node.next
+    def parse(parent_node)
+      # run element parser
+      match=parser.call(parent_node)
+
+      # Negative patterns (PEG: !element)
+      match=match ? nil : EmptyNode.new(parent_node) if negative
+
+      # Optional patterns (PEG: element?)
+      match=EmptyNode.new(parent_node) if !match && optional
+
+      # Could-match patterns (PEG: &element)
+      match.match_length=0 if match && could_match
+
+      # return match
+      match
+    end
+
+    private
+
+    # initialize PatternElement based on the type of: match
+    def init(match)
+      self.match=match
+      case match
+      when TrueClass then init_true
+      when String then    init_string match
+      when Regexp then    init_regex match
+      when Symbol then    init_rule match
+      when Hash then      init_hash match
+      else                raise "invalid pattern type: #{match.inspect}"
+      end
+    end
+
+    # "true" parser always matches the empty string
+    def init_true
+      self.parser=lambda {|parent_node| EmptyNode.new(parent_node)}
+    end
+
+    # initialize PatternElement as a parser that matches exactly the string specified
+    def init_string(string)
+      self.parser=lambda do |parent_node|
+        if parent_node.src[parent_node.next,string.length]==string
+          TerminalNode.new(parent_node,string.length,string)
+        end
+      end
+      self.terminal=true
+    end
+
+    # initialize PatternElement as a parser that matches the given regex
+    def init_regex(regex)
+      optimized_regex=/\A#{regex}/  # anchor the search
+      self.parser=lambda do |parent_node|
+        offset=parent_node.next
+        if parent_node.src[offset..-1].index(optimized_regex)==0
+          range=$~.offset(0)
+          TerminalNode.new(parent_node,range[1]-range[0],regex)
+        end
+      end
+      self.terminal=true
+    end
+
+    # initialize PatternElement as a parser that matches a named sub-rule
+    def init_rule(rule_name)
+      rule_name.to_s[/^([^?!]*)([?!])?$/]
+      rule_name=$1.to_sym
+      option=$2
+      match_rule=rule_variant.rule.parser.rules[rule_name]
+      raise "no rule for #{rule_name}" unless match_rule
+
+      self.parser = lambda {|parent_node| match_rule.parse(parent_node)}
+      self.name   = rule_name
+      case option
+      when "?"  then self.optional=true
+      when "!"  then self.negative=true
+      end
+    end
+
+    # initialize the PatternElement from hashed parameters
+    def init_hash(hash)
+      if hash[:parser]
+        self.parser=hash[:parser]
+      elsif hash[:many]
+        init_many hash
+      elsif hash[:match]
+        init hash[:match]
+      else
+        raise "extended-options patterns (specified by a hash) must have either :parser=> or a :match=> set"
+      end
+
+      self.name = hash[:as] || self.name
+      self.optional ||= hash[:optional] || hash[:optionally]
+      self.could_match ||= hash[:could]
+      self.negative ||= hash[:dont]
+    end
+
+    # initialize the PatternElement as a many-parser from hashed parameters (hash[:many] is assumed to be set)
+    def init_many(hash)
+      # generate single_parser
+      init hash[:many]
+      single_parser=parser
+
+      # generate delimiter_pattern_element
+      delimiter_pattern_element= hash[:delimiter] && PatternElement.new(hash[:delimiter],rule_variant)
+
+      # generate post_delimiter_element
+      post_delimiter_element=hash[:post_delimiter] && case hash[:post_delimiter]
+      when TrueClass then delimiter_pattern_element
+      else PatternElement.new(hash[:post_delimiter],rule_variant)
+      end
+
+      # generate many-parser
+      self.parser= lambda do |parent_node|
+        last_match=single_parser.call(parent_node)
+        many_node=ManyNode.new(parent_node)
+
+        if delimiter_pattern_element
+          # delimited matching
+          while last_match
+            many_node<<last_match
+
+            #match delimiter
+            delimiter_match=delimiter_pattern_element.parse(many_node)
+            break unless delimiter_match
+            many_node.delimiter_matches<<delimiter_match
+
+            #match next
+            last_match=single_parser.call(many_node)
+          end
+        else
+          # not delimited matching
+          while last_match
+            many_node<<last_match
+            last_match=single_parser.call(many_node)
+          end
+        end
+
+        # success only if we have at least one match
+        return nil unless many_node.length>0
+
+        # pop the post delimiter matched with delimiter_pattern_element
+        many_node.delimiter_matches.pop if many_node.length==many_node.delimiter_matches.length
+
+        # If post_delimiter is requested, many_node and delimiter_matches will be the same length
+        if post_delimiter_element
+          post_delimiter_match=post_delimiter_element.parse(many_node)
+
+          # fail if post_delimiter didn't match
+          return nil unless post_delimiter_match
+          many_node.delimiter_matches<<post_delimiter_match
+        end
+
+        many_node
+      end
+    end
+  end
+end

data/test/test_bb.rb

@@ -68,7 +68,7 @@ class BBTests < TestHelper
 
   def test_regex
     parser=new_parser do
-      rule :foo, [/[0-9]+/]
+      rule :foo, /[0-9]+/
     end
 
     %w{ 0 1 10 123 1001 }.each do |numstr|
@@ -76,6 +76,16 @@ class BBTests < TestHelper
     end
   end
 
+  def test_regex_offset
+    parser=new_parser do
+      rule :foo, /[0-9]+/
+      rule :foo, "hi", /[0-9]+/
+    end
+
+    assert_equal 1,parser.parse("123").matches.length
+    assert_equal 2,parser.parse("hi123").matches.length
+  end
+
   def test_optional
     parser=new_parser do
       rule :foo, ["foo", :bar?]
@@ -311,10 +321,11 @@ class BBTests < TestHelper
 
   def test_poly_post_delimiter
     parser=new_parser do
-      rule :foo, many?("foo",/ +/,true).as(:foo), "end"
+      rule :foo, many?("foo",/ +/,true).as(:foo), match("end").as(:end)
     end
 
-    assert_equal 0,parser.parse("end").foo.length
+    assert_equal nil,parser.parse("end").foo
+    assert_equal "end",parser.parse("end").end.to_s
     assert_equal nil,parser.parse(" end")
     assert_equal nil,parser.parse("foofoo end")
     assert_equal ["foo"], parser.parse("foo end").foo.collect {|f| f.text}
@@ -378,10 +389,24 @@ class BBTests < TestHelper
     end
     parser.parse "-"
   end
-end
 
-tests=BBTests.new
 
+  def regex_performance
+    parser=new_parser do
+      rule :foo, many(:element)
+      rule :element, /[0-9]+/
+      rule :element, "a"
+    end
 
+    str=("a"*10000)+"1"
+    start_time=Time.now
+    res=parser.parse(str)
+    end_time=Time.now
+    puts "time for matching string of length #{str.length}: #{((end_time-start_time)*1000).to_i}ms"
+    puts "parse tree size: #{res.element.length}"
+    assert res
+  end
+end
 
+tests=BBTests.new
 tests.run_tests(ARGV.length>0 && ARGV)

metadata

@@ -1,69 +1,53 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: babel_bridge
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 1
-  - 1
-  version: 0.1.1
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Shane Brinkman-Davis
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-11-28 00:00:00 -08:00
-default_executable: 
+date: 2010-11-28 00:00:00.000000000Z
 dependencies: []
-
-description: Babel Bridge let's you generate parsers 100% in Ruby code. It is a memoizing Parsing Expression Grammar (PEG) generator like Treetop, but it doesn't require special file-types or new syntax. Overall focus is on simplicity and usability over performance.
+description: ! "Babel Bridge is an object oriented parser generator for parsing expression
+  grammars (PEG). \nGenerate memoizing packrat parsers 100% in Ruby code with a simple
+  embedded DSL.\n"
 email: shanebdavis@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - README
 - babel_bridge.gemspec
 - test/test_bb.rb
 - test/test_helper.rb
 - lib/babel_bridge.rb
 - lib/nodes.rb
-has_rdoc: true
+- lib/pattern_element.rb
 homepage: http://babel-bridge.rubyforge.org
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: babel-bridge
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: A Ruby-based parser-generator based on Parsing Expression Grammars.
 test_files: []
-