babel_bridge 0.4.0 → 0.4.1

data/lib/nodes/empty_node.rb

@@ -14,14 +14,26 @@ class EmptyNode < Node
     "EmptyNode" unless options[:simple]
   end
 
-  # EmptyNodes should always match at the beginning of the whitespace range
-  def node_init(parent_or_parser)
-    super
-    self.offset = preceding_whitespace_range.first
-    self.preceding_whitespace_range = match_range
+  def matches; [self]; end
+
+end
+
+class RollbackWhitespaceNode < Node
+  def inspect(options={})
+    "RollbackWhitespace" unless options[:simple]
   end
 
   def matches; [self]; end
 
+  def initialize(parent)
+    super
+    self.match_length = 0
+    self.offset = parent.postwhitespace_range.first
+  end
+
+  def postwhitespace_range
+    @postwhitespace_range ||= offset_after_match .. offset_after_match-1
+  end
+
 end
 end

data/lib/nodes/many_node.rb

@@ -15,7 +15,7 @@ class ManyNode < NonTerminalNode
 
   def separate_delimiter_matches
     count = 0
-    @matches, @delimiter_matches = @matches.partition {|el| count+=1;(count%2==1)}
+    @matches, @delimiter_matches = matches.partition {|el| count+=1;(count%2==1)}
     @delimiter_matches = @delimiter_matches[0..@matches.length-2] #delimiter_matches should be exactly 1 shorter
     update_match_length
   end

data/lib/nodes/node.rb

@@ -7,7 +7,12 @@ end
 
 # base class for all parse-tree nodes
 class Node
-  attr_accessor :src,:offset,:match_length,:parent,:parser,:preceding_whitespace_range
+  attr_accessor :src,:offset,:match_length,:parent,:parser,:prewhitespace_range
+
+  # no_postwhitespace is used when parsing to temporarilly rollback the preceeding whitespace while
+  # attempting to match an ignore_whitespace pattern.
+  # It should always be false again once parsing completes or fails.
+  attr_accessor :no_postwhitespace
 
   def whitespace_regexp
     parser.whitespace_regexp
@@ -22,10 +27,27 @@ class Node
     offset..(offset+match_length-1)
   end
 
-  def trailing_whitespace_range
+  def postwhitespace_range_without_no_postwhitespace
     parser.white_space_range offset_after_match
   end
 
+  def postwhitespace_range
+    r = postwhitespace_range_without_no_postwhitespace
+    no_postwhitespace ? r.first..r.first-1 : r
+  end
+
+  def postwhitespace
+    src[postwhitespace_range]
+  end
+
+  def prewhitespace
+    src[prewhitespace_range]
+  end
+
+  # called when a ruled is matched
+  def matched
+  end
+
   def to_s
     text
   end
@@ -45,7 +67,7 @@ class Node
       self.parent=parent_or_parser
       self.parser=parent.parser
       self.offset=parent.next
-      self.preceding_whitespace_range=parent.trailing_whitespace_range
+      self.prewhitespace_range=parent.postwhitespace_range
       self.src=parent.src
       raise "parent node does not have parser set" unless parser
     else
@@ -73,7 +95,7 @@ class Node
   #********************
   # info methods
   #********************
-  def next; trailing_whitespace_range.last+1 end # index of first character after match and any trailing whitespace
+  def next; postwhitespace_range.last+1 end # index of first character after match and any trailing whitespace
   def text; src[match_range] end  # the substring in src matched
 
   # length returns the number of sub-nodes

data/lib/nodes/non_terminal_node.rb

@@ -8,18 +8,18 @@ module BabelBridge
 # rule node
 # subclassed automatically by parser.rule for each unique non-terminal
 class NonTerminalNode < Node
+  attr_accessor :last_non_empty_node
 
-  def trailing_whitespace_range
-    if matches.length == 0
-      preceding_whitespace_range || (0..-1)
+  def postwhitespace_range_without_no_postwhitespace
+    if  last_non_empty_node
+      last_non_empty_node.postwhitespace_range
     else
-      matches[-1].trailing_whitespace_range
+      prewhitespace_range || (0..-1)
     end
   end
 
   def update_match_length
-    m = matches[-1]
-    @match_length = m ? m.offset_after_match - offset : 0
+    @match_length = last_non_empty_node ? last_non_empty_node.offset_after_match - offset : 0
   end
 
   #*****************************
@@ -34,7 +34,8 @@ class NonTerminalNode < Node
     matches.length
   end
 
-  def <<(node)
+  def add_match(node)
+    @last_non_empty_node = node unless node.kind_of?(EmptyNode)
     matches<<node
     update_match_length
   end

data/lib/nodes/rule_node.rb

@@ -93,7 +93,7 @@ class RuleNode < NonTerminalNode
   # adds a match with name (optional)
   def add_match(match,name=nil)
     reset_matches_by_name
-    matches<<match
+    super match
     match_names<<name
 
     update_match_length

data/lib/nodes/terminal_node.rb

@@ -8,7 +8,7 @@ module BabelBridge
 # used for String and Regexp PatternElements
 # not subclassed
 class TerminalNode < Node
-  attr_accessor :pattern, :trailing_whitespace_offset
+  attr_accessor :pattern, :postwhitespace_offset
   def initialize(parent,range,pattern)
     node_init(parent)
     self.offset = range.min

data/lib/parser.rb

@@ -18,8 +18,6 @@ class Parser
     #
     # rules can be specified as:
     #   rule :name, to_match1, to_match2, etc...
-    #or
-    #   rule :name, [to_match1, to_match2, etc...]
     #
     # Can define rules INSIDE class:
     #   class MyParser < BabelBridge::Parser
@@ -41,9 +39,8 @@ class Parser
     # This allows you to add whatever functionality you want to a your nodes in the final parse tree.
     # Also note you can override the post_match method. This allows you to restructure the parse tree as it is parsed.
     def rule(name,*pattern,&block)
-      pattern=pattern[0] if pattern[0].kind_of?(Array)
-      rule=self.rules[name]||=Rule.new(name,self)
-      self.root_rule||=name
+      rule = self.rules[name] ||= Rule.new(name,self)
+      self.root_rule ||= name
       rule.add_variant(pattern,&block)
     end
 
@@ -140,9 +137,7 @@ class Parser
     def match(*args) PatternElementHash.new.match(*args) end
     def match!(*args) PatternElementHash.new.dont.match(*args) end
 
-    # if ignore_whitespace is used, after every TerminalNode, all whitespace is consumed. Wrapping include_whitespace around a pattern-element allows you to explicitly match the preceeding whitespace for that one element.
-    # NOTE: you can ALWAYS explicitly match any trailing whitespace
-    def include_whitespace(*args) PatternElementHash.new.include_whitespace.match(*args) end
+    def rewind_whitespace; PatternElementHash.new.rewind_whitespace end
 
     def dont; PatternElementHash.new.dont end
     def optionally; PatternElementHash.new.optionally end

data/lib/pattern_element.rb

@@ -21,7 +21,7 @@ end
 #   :optional
 class PatternElement
   attr_accessor :parser,:optional,:negative,:name,:terminal,:could_match
-  attr_accessor :match,:rule_variant,:include_whitespace
+  attr_accessor :match,:rule_variant,:rewind_whitespace
 
   #match can be:
   # true, Hash, Symbol, String, Regexp
@@ -42,21 +42,23 @@ class PatternElement
 
   # attempt to match the pattern defined in self.parser in parent_node.src starting at offset parent_node.next
   def parse(parent_node)
+    return RollbackWhitespaceNode.new(parent_node) if rewind_whitespace
+
     # run element parser
-    match=parser.call(parent_node)
+    match = parser.call(parent_node)
 
     # Negative patterns (PEG: !element)
-    match=match ? nil : EmptyNode.new(parent_node) if negative
+    match = match ? nil : EmptyNode.new(parent_node) if negative
 
     # Optional patterns (PEG: element?)
-    match=EmptyNode.new(parent_node) if !match && optional
+    match = EmptyNode.new(parent_node) if !match && optional
 
     # Could-match patterns (PEG: &element)
-    match.match_length=0 if match && could_match
+    match.match_length = 0 if match && could_match
 
     if !match && terminal
       # log failures on Terminal patterns for debug output if overall parse fails
-      parent_node.parser.log_parsing_failure(match_start_index(parent_node),:pattern=>self.match,:node=>parent_node)
+      parent_node.parser.log_parsing_failure(parent_node.next,:pattern=>self.match,:node=>parent_node)
     end
 
     # return match
@@ -67,7 +69,7 @@ class PatternElement
 
   # initialize PatternElement based on the type of: match
   def init(match)
-    self.match=match
+    self.match = match
     case match
     when TrueClass then init_true
     when String then    init_string match
@@ -83,15 +85,6 @@ class PatternElement
     self.parser=lambda {|parent_node| EmptyNode.new(parent_node)}
   end
 
-  def match_start_index(parent_node)
-    if include_whitespace
-      parent_node.trailing_whitespace_range.first
-    else
-      parent_node.next
-    end
-  end
-
-
   # initialize PatternElement as a parser that matches exactly the string specified
   def init_string(string)
     init_regex Regexp.escape(string)
@@ -101,7 +94,7 @@ class PatternElement
   def init_regex(regex)
     optimized_regex=/\A#{regex}/  # anchor the search
     self.parser=lambda do |parent_node|
-      offset = match_start_index(parent_node)
+      offset = parent_node.next
       if parent_node.src[offset..-1].index(optimized_regex)==0
         range=$~.offset(0)
         range = (range.min+offset)..(range.max+offset)
@@ -114,16 +107,16 @@ class PatternElement
   # initialize PatternElement as a parser that matches a named sub-rule
   def init_rule(rule_name)
     rule_name.to_s[/^([^?!]*)([?!])?$/]
-    rule_name=$1.to_sym
-    option=$2
-    match_rule=rule_variant.rule.parser.rules[rule_name]
+    rule_name = $1.to_sym
+    option = $2
+    match_rule = rule_variant.rule.parser.rules[rule_name]
     raise "no rule for #{rule_name}" unless match_rule
 
     self.parser = lambda {|parent_node| match_rule.parse(parent_node)}
-    self.name   = rule_name
+    self.name = rule_name
     case option
-    when "?"  then self.optional=true
-    when "!"  then self.negative=true
+    when "?"  then self.optional = true
+    when "!"  then self.negative = true
     end
   end
 
@@ -135,6 +128,9 @@ class PatternElement
       init_many hash
     elsif hash[:match]
       init hash[:match]
+    elsif hash[:rewind_whitespace]
+      self.rewind_whitespace = true
+      return
     else
       raise "extended-options patterns (specified by a hash) must have either :parser=> or a :match=> set"
     end
@@ -143,7 +139,6 @@ class PatternElement
     self.optional ||= hash[:optional] || hash[:optionally]
     self.could_match ||= hash[:could]
     self.negative ||= hash[:dont]
-    self.include_whitespace ||= hash[:include_whitespace]
   end
 
   # initialize the PatternElement as a many-parser from hashed parameters (hash[:many] is assumed to be set)
@@ -165,12 +160,12 @@ class PatternElement
           #match primary
           match = single_parser.call many_node
           break unless match
-          many_node << match
+          many_node.add_match match
 
           #match delimiter
           delimiter_match = delimiter_pattern_element.parse many_node
           break unless delimiter_match
-          many_node << delimiter_match
+          many_node.add_match delimiter_match
         end
         many_node.separate_delimiter_matches
       else
@@ -178,7 +173,7 @@ class PatternElement
         while true
           match = single_parser.call many_node
           break unless match
-          many_node << match
+          many_node.add_match match
         end
       end
 

data/lib/rule_variant.rb

@@ -26,6 +26,7 @@ class RuleVariant
 
       # if parse failed
       return if !match
+      match.matched
 
       # parse succeeded, add to node and continue
       node.add_match(match,pe.name)
@@ -34,6 +35,6 @@ class RuleVariant
   end
 
   def inspect; pattern.collect {|a| a.inspect}.join(', '); end
-  def to_s; "variant_class: #{variant_node_class}, pattern: #{inspect}"; end  
+  def to_s; "variant_class: #{variant_node_class}, pattern: #{inspect}"; end
+end
 end
-end

data/lib/version.rb

@@ -1,3 +1,3 @@
 module BabelBridge
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/spec/bb_spec.rb

@@ -13,9 +13,10 @@ describe BabelBridge do
   #options
   #   :parser
   #   :failure_ok
-  def test_parse(string,options={})
+  def test_parse(string,options={},&block)
     parser = options[:parser] || @parser
     res = parser.parse(string)
+    yield res if res && block
     if options[:should_fail_at]
       res.should == nil
       parser.failure_index.should == options[:should_fail_at]
@@ -69,7 +70,7 @@ describe BabelBridge do
       ignore_whitespace
 
       rule :pair, :statement, :end_statement, :statement
-      rule :end_statement, include_whitespace(/([\t ]*[\n;])+/)
+      rule :end_statement, rewind_whitespace, /([\t ]*[\n;])+/
       rule :statement, "0"
     end
 
@@ -89,7 +90,7 @@ describe BabelBridge do
       ignore_whitespace
 
       rule :pair, :statement, :end_statement, :statement
-      rule :end_statement, include_whitespace(/([\t ]*[\n;])+/)
+      rule :end_statement, rewind_whitespace, /([\t ]*[\n;])+/
       rule :statement, "0", :one?, :one?, :one?
       rule :one, "1"
     end
@@ -106,7 +107,7 @@ describe BabelBridge do
     new_parser do
       ignore_whitespace
       rule :statements, many(:statement,:end_statement)
-      rule :end_statement, include_whitespace(/([\t ]*[;\n])+/)
+      rule :end_statement, rewind_whitespace, /([\t ]*[;\n])+/
       rule :statement, "0"
     end
 
@@ -129,7 +130,7 @@ describe BabelBridge do
     test_parse "foo-bar", :should_fail_at => 3
   end
 
-  it "should work to have many many parsing" do
+  it "should work to have many-many parsing" do
     new_parser do
       rule :top, many(:bottom,";")
       rule :bottom, many("0",",")
@@ -143,11 +144,11 @@ describe BabelBridge do
     test_parse "0,0,0;0;0,0,0"
   end
 
-  it "should work to have many many parsing with whitespace tricks" do
+  it "should work to have many parsing with whitespace tricks" do
     new_parser do
       ignore_whitespace
       rule :statements, many(:statement,:end_statement)
-      rule :end_statement, include_whitespace(/([\t ]*[;\n])+/)
+      rule :end_statement, rewind_whitespace, /([\t ]*[;\n])+/
       rule :statement, :bin_op
       binary_operators_rule :bin_op, :int, ["**", [:/, :*], [:+, "-"]], :right_operators => ["**"]
       rule :int, /\d+/
@@ -161,4 +162,50 @@ describe BabelBridge do
 ENDCODE
   end
 
+  it "should work to rewind_whitespace, :rule" do
+    new_parser do
+      ignore_whitespace
+      rule :all, :identifier, :parameter?, :identifier do
+        def to_model
+          [[identifier[0].to_sym, parameter && parameter.to_sym], identifier[1].to_sym]
+        end
+      end
+      rule :parameter, rewind_whitespace, /[ \t]*/, rewind_whitespace, :identifier
+      rule :identifier, /[_a-zA-Z][_a-zA-Z0-9]*/
+    end
+
+    test_parse("fred\nbar") {|parsed|parsed.to_model.should == [[:fred,nil],:bar]}
+    test_parse("fred foo\nbar") {|parsed|parsed.to_model.should == [[:fred,:foo],:bar]}
+  end
+
+  it "should work to rewind_whitespace, many" do
+    new_parser do
+      ignore_whitespace
+      rule :all, :identifier, :parameters?, :identifier do
+        def to_model
+          [[identifier[0].to_sym, parameters && parameters.to_s], identifier[1].to_sym]
+        end
+      end
+      rule :parameters, rewind_whitespace, /[ \t]*/, rewind_whitespace, many(:identifier,",")
+      rule :identifier, /[_a-zA-Z][_a-zA-Z0-9]*/
+    end
+
+    test_parse("fred\nbar")           {|parsed| parsed.to_model.should==[[:fred,nil],:bar]}
+    test_parse("fred foo\nbar")       {|parsed| parsed.to_model.should==[[:fred,"foo"],:bar]}
+    test_parse("fred foo, bar\nbar")  {|parsed| parsed.to_model.should==[[:fred,"foo, bar"],:bar]}
+  end
+
+  it "dont.match shouldn't consume any whitespace" do
+    new_parser do
+      ignore_whitespace
+      rule :statements, :statement, "bar"
+      rule :statement, :identifier, :parameters?
+      rule :parameters, rewind_whitespace, / */, rewind_whitespace, :identifier
+      rule :identifier, dont.match("end"), /[_a-zA-Z][_a-zA-Z0-9]*/
+    end
+
+    test_parse("fred\nbar")
+    test_parse("fred foo\nbar")
+  end
+
 end

data/test/test_bb.rb

@@ -58,7 +58,7 @@ class BBTests < TestHelper
 
   def test_foo
     parser=new_parser do
-      rule :foo, ["foo"]
+      rule :foo, "foo"
     end
 
     assert p=parser.parse("foo")
@@ -88,8 +88,8 @@ class BBTests < TestHelper
 
   def test_optional
     parser=new_parser do
-      rule :foo, ["foo", :bar?]
-      rule :bar, ["bar"]
+      rule :foo, "foo", :bar?
+      rule :bar, "bar"
     end
 
     assert parser.parse("foo")
@@ -107,8 +107,8 @@ class BBTests < TestHelper
 
   def test_optional_middle
     parser=new_parser do
-      rule :foo, ["foo", :bar?, "foo"]
-      rule :bar, ["bar"]
+      rule :foo, "foo", :bar?, "foo"
+      rule :bar, "bar"
     end
 
     assert parser.parse("foofoo")
@@ -117,8 +117,8 @@ class BBTests < TestHelper
 
   def test_greedy_optional_middle
     parser=new_parser do
-      rule :foo, ["foo", :bar?, "foo"]
-      rule :bar, ["foo"]
+      rule :foo, "foo", :bar?, "foo"
+      rule :bar, "foo"
     end
 
     assert_nil parser.parse("foofoo")
@@ -127,8 +127,8 @@ class BBTests < TestHelper
 
   def test_not
     parser=new_parser do
-      rule :foo, ["foo", :bar!]
-      rule :bar, ["bar"]
+      rule :foo, "foo", :bar!
+      rule :bar, "bar"
     end
 
     assert_nil parser.parse("foofud") # this should fail because it doesn't match the entire input
@@ -139,7 +139,7 @@ class BBTests < TestHelper
 
   def test_recursive
     parser=new_parser do
-      rule :foo, ["foo", :foo?]
+      rule :foo, "foo", :foo?
     end
 
     assert parser.parse("foo")
@@ -153,8 +153,8 @@ class BBTests < TestHelper
     v1=nil
     v2=nil
     parser=new_parser do
-      v1=rule :foo, ["foo"]
-      v2=rule :foo, ["bar"]
+      v1=rule :foo, "foo"
+      v2=rule :foo, "bar"
     end
 
     assert r1=parser.parse("foo")
@@ -165,8 +165,8 @@ class BBTests < TestHelper
 
   def test_add
     parser=new_parser do
-      rule :add, [:number,"+",:number]
-      rule :number, [/[0-9]+/]
+      rule :add, :number, "+", :number
+      rule :number, /[0-9]+/
     end
 
     assert parser.parse("1+1")
@@ -175,7 +175,7 @@ class BBTests < TestHelper
 
   def test_method
     parser=new_parser do
-      rule :number, [/[0-9]+/] do
+      rule :number, /[0-9]+/ do
         def number
           text.to_i
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: babel_bridge
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
   prerelease: 
 platform: ruby
 authors: