babel_bridge 0.2.0 → 0.3.0

data/lib/nodes/non_terminal_node.rb

@@ -0,0 +1,117 @@
+=begin
+Copyright 2011 Shane Brinkman-Davis
+See README for licence information.
+http://babel-bridge.rubyforge.org/
+=end
+
+module BabelBridge
+# non-terminal node
+# subclassed automatically by parser.rule for each unique non-terminal
+class NonTerminalNode < Node
+  attr_accessor :matches,:match_names
+
+  def match_names
+    @match_names ||= []
+  end
+  def matches
+    @matches ||= []
+  end
+
+  # length returns the number of sub-nodes
+  def length
+    matches.length
+  end
+
+  def matches_by_name
+    @matches_by_name||= begin
+      raise "matches.length #{matches.length} != match_names.length #{match_names.length}" unless matches.length==match_names.length
+      mbn={}
+      mn=match_names
+      matches.each_with_index do |match,i|
+        name=mn[i]
+        next unless name
+        if current=mbn[name] # name already used
+          # convert to MultiMatchesArray if not already
+          mbn[name]=MultiMatchesArray.new([current]) if !current.kind_of? MultiMatchesArray
+          # add to array
+          mbn[name]<<match
+        else
+          mbn[name]=match
+        end
+      end
+      mbn
+    end
+  end
+
+  def inspect(options={})
+    return "#{self.class}" if matches.length==0
+    matches_inspected=matches.collect{|a|a.inspect(options)}.compact
+    if matches_inspected.length==0 then nil
+    elsif matches_inspected.length==1
+      m=matches_inspected[0]
+      ret="#{self.class} > "+matches_inspected[0]
+      if options[:simple]
+        ret=if m["\n"] then m
+        else
+          # just show the first and last nodes in the chain
+          ret.gsub(/( > [A-Z][a-zA-Z0-9:]+ > (\.\.\. > )?)/," > ... > ")
+        end
+      end
+      ret
+    else
+      (["#{self.class}"]+matches_inspected).join("\n").gsub("\n","\n  ")
+    end
+  end
+
+  #********************
+  # alter methods
+  #********************
+  def reset_matches_by_name
+    @matches_by_name=nil
+  end
+
+  # defines where to forward missing methods to; override for custom behavior
+  def forward_to(method_name)
+    matches.each {|m| return m if m.respond_to?(method_name)}
+    nil
+  end
+
+  def respond_to?(method_name)
+    super ||
+    matches_by_name[method_name] ||
+    forward_to(method_name)
+  end
+
+  def method_missing(method_name, *args)  #method_name is a symbol
+    unless matches_by_name.has_key? method_name
+      if f=forward_to(method_name)
+        return f.send(method_name,*args)
+      end
+      raise "#{self.class}: missing method #{method_name.inspect} / doesn't match named pattern element: #{matches_by_name.keys.inspect}"
+    end
+    case ret=matches_by_name[method_name]
+    when EmptyNode then nil
+    else ret
+    end
+  end
+
+  # adds a match with name (optional)
+  # returns self so you can chain add_match or concat methods
+  def add_match(match,name=nil)
+    reset_matches_by_name
+    matches<<match
+    match_names<<name
+
+    self.match_length=match.next - offset
+    self
+  end
+
+  # concatinate all matches from another node
+  # returns self so you can chain add_match or concat methods
+  def concat(node)
+    names=node.match_names
+    node.matches.each_with_index { |match,i| add_match(match,names[i])}
+    self
+  end
+end
+end

data/lib/nodes/terminal_node.rb

@@ -0,0 +1,38 @@
+=begin
+Copyright 2011 Shane Brinkman-Davis
+See README for licence information.
+http://babel-bridge.rubyforge.org/
+=end
+
+module BabelBridge
+# used for String and Regexp PatternElements
+# not subclassed
+class TerminalNode < Node
+  attr_accessor :pattern
+  def initialize(parent,match_length,pattern)
+    node_init(parent)
+    self.match_length=match_length
+    self.pattern=pattern
+    @ignore_whitespace = parser.ignore_whitespace?
+    consume_trailing_whitespace if @ignore_whitespace
+  end
+
+  def consume_trailing_whitespace
+    offset = self.next
+    if src[offset..-1].index(/\A\s*/)==0
+      range = $~.offset(0)
+      self.match_length += range[1]-range[0]
+    end
+  end
+
+  def to_s
+    @ignore_whitespace ? text.strip : text
+  end
+
+  def inspect(options={})
+    "#{text.inspect}" unless options[:simple] && text[/^\s*$/] # if simple && node only matched white-space, return nil
+  end
+
+  def matches; [self]; end
+end
+end

data/lib/parser.rb

@@ -0,0 +1,285 @@
+module BabelBridge
+# primary object used by the client
+# Used to generate the grammer with .rule methods
+# Used to parse with .parse
+class Parser
+
+  # Parser sub-class grammaer definition
+  # These methods are used in the creation of a Parser Sub-Class to define
+  # its grammar
+  class <<self
+    attr_accessor :rules,:module_name,:root_rule
+
+    def rules
+      @rules||={}
+    end
+
+    # Add a rule to the parser
+    #
+    # rules can be specified as:
+    #   rule :name, to_match1, to_match2, etc...
+    #or
+    #   rule :name, [to_match1, to_match2, etc...]
+    #
+    # Can define rules INSIDE class:
+    #   class MyParser < BabelBridge::Parser
+    #     rule :name, to_match1, to_match2, etc...
+    #   end
+    #
+    # Or can define rules OUTSIDE class:
+    #   class MyParser < BabelBridge::Parser
+    #   end
+    #   MyParser.rule :name, to_match1, to_match2, etc...
+    #
+    # The first rule added is the root-rule for the parser.
+    # You can override by: 
+    #   class MyParser < BabelBridge::Parser
+    #     root_rule = :new_root_rool
+    #   end
+    #
+    # The block is executed in the context of the rule-varient's node type, a subclass of: NonTerminalNode
+    # This allows you to add whatever functionality you want to a your nodes in the final parse tree.
+    # Also note you can override the post_match method. This allows you to restructure the parse tree as it is parsed.
+    def rule(name,*pattern,&block)
+      pattern=pattern[0] if pattern[0].kind_of?(Array)
+      rule=self.rules[name]||=Rule.new(name,self)
+      self.root_rule||=name
+      rule.add_variant(pattern,&block)
+    end
+
+    # options
+    # => right_operators: list of all operators that should be evaluated right to left instead of left-to-write
+    #       typical example is the "**" exponentiation operator which should be evaluated right-to-left.
+    def binary_operators_rule(name,elements_pattern,operators,options={},&block)
+      right_operators = options[:right_operators]
+      rule(name,many(elements_pattern,Tools::array_to_or_regexp(operators))) do 
+        self.class_eval &block if block
+        class <<self
+          attr_accessor :operators_from_rule, :right_operators
+          def operator_processor
+            @operator_processor||=BinaryOperatorProcessor.new(operators_from_rule,self,right_operators)
+          end
+        end
+        self.right_operators = right_operators
+        self.operators_from_rule = operators
+
+        def operator
+          @operator||=operator_node.to_s.to_sym
+        end
+
+        # Override the post_match method to take the results of the "many" match
+        # and restructure it into a binary tree of nodes based on the precidence of
+        # the "operators".
+        # TODO - I think maybe post_match should be run after the whole tree matches. If not, will this screw up caching?
+        def post_match
+          many_match = matches[0]
+          operands = many_match.matches
+          operators = many_match.delimiter_matches
+          # TODO - now! take many_match.matches and many_match.delimiter_matches, mishy-mashy, and make the super-tree!
+          self.class.operator_processor.generate_tree operands, operators, parent
+        end
+      end
+    end
+
+    def node_class(name,&block)
+      klass=self.rules[name].node_class
+      return klass unless block
+      klass.class_eval &block
+    end
+
+    def [](i)
+      rules[i]
+    end
+
+    # rule can be symbol-name of one of the rules in rules_array or one of the actual Rule objects in that array
+    def root_rule=(rule)
+      raise "Symbol required" unless rule.kind_of?(Symbol)
+      raise "rule #{rule.inspect} not found" unless rules[rule]
+      @root_rule=rule
+    end
+
+    def ignore_whitespace
+      @ignore_whitespace = true
+    end
+
+    def ignore_whitespace?
+      @ignore_whitespace
+    end
+  end
+
+  def ignore_whitespace?
+    self.class.ignore_whitespace?
+  end
+
+  #*********************************************
+  # pattern construction tools
+  #
+  # Ex:
+  #   # match 'keyword'
+  #   # (succeeds if keyword is matched; advances the read pointer)
+  #   rule :sample_rule, "keyword"
+  #   rule :sample_rule, match("keyword")
+  #
+  #   # don't match 'keyword'
+  #   # (succeeds only if keyword is NOT matched; does not advance the read pointer)
+  #   rule :sample_rule, match!("keyword")
+  #   rule :sample_rule, dont.match("keyword")
+  #
+  #   # optionally match 'keyword'
+  #   # (always succeeds; advances the read pointer if keyword is matched)
+  #   rule :sample_rule, match?("keyword")
+  #   rule :sample_rule, optionally.match("keyword")
+  #
+  #   # ensure we could match 'keyword'
+  #   # (succeeds only if keyword is matched, but does not advance the read pointer)
+  #   rule :sample_rule, could.match("keyword")
+  #
+  #*********************************************
+  class <<self
+    def many(m,delimiter=nil,post_delimiter=nil) PatternElementHash.new.match.many(m).delimiter(delimiter).post_delimiter(post_delimiter) end
+    def many?(m,delimiter=nil,post_delimiter=nil) PatternElementHash.new.optionally.match.many(m).delimiter(delimiter).post_delimiter(post_delimiter) end
+    def many!(m,delimiter=nil,post_delimiter=nil) PatternElementHash.new.dont.match.many(m).delimiter(delimiter).post_delimiter(post_delimiter) end
+
+    def match?(*args) PatternElementHash.new.optionally.match(*args) end
+    def match(*args) PatternElementHash.new.match(*args) end
+    def match!(*args) PatternElementHash.new.dont.match(*args) end
+
+    def dont; PatternElementHash.new.dont end
+    def optionally; PatternElementHash.new.optionally end
+    def could; PatternElementHash.new.could end
+  end
+
+
+  #*********************************************
+  #*********************************************
+  # parser instance implementation
+  # this methods are used for each actual parse run
+  # they are tied to an instnace of the Parser Sub-class to you can have more than one
+  # parser active at a time
+  attr_accessor :failure_index
+  attr_accessor :expecting_list
+  attr_accessor :src
+  attr_accessor :parse_cache
+
+  def initialize
+    reset_parser_tracking
+  end
+
+  def reset_parser_tracking
+    self.src=nil
+    self.failure_index=0
+    self.expecting_list={}
+    self.parse_cache={}
+  end
+
+  def cached(rule_class,offset)
+    (parse_cache[rule_class]||={})[offset]
+  end
+
+  def cache_match(rule_class,match)
+    (parse_cache[rule_class]||={})[match.offset]=match
+  end
+
+  def cache_no_match(rule_class,offset)
+    (parse_cache[rule_class]||={})[offset]=:no_match
+  end
+
+  def log_parsing_failure(index,expecting)
+    if index>failure_index
+      key=expecting[:pattern]
+      @expecting_list={key=>expecting}
+      @failure_index = index
+    elsif index == failure_index
+      key=expecting[:pattern]
+      self.expecting_list[key]=expecting
+    else
+      # ignored
+    end
+  end
+
+  def parse(src,offset=0,rule=nil)
+    reset_parser_tracking
+    @start_time=Time.now
+    self.src=src
+    root_node=RootNode.new(self)
+    raise "No root rule defined." unless rule || self.class.root_rule
+    ret=self.class[rule||self.class.root_rule].parse(root_node)
+    unless rule
+      if ret
+        if ret.next<src.length # parse only succeeds if the whole input is matched
+          @parsing_did_not_match_entire_input=true
+          @failure_index=ret.next
+          ret=nil
+        else
+          reset_parser_tracking
+        end
+      end
+    end
+    @end_time=Time.now
+    ret
+  end
+
+  def parse_time
+    @end_time-@start_time
+  end
+
+  def parse_and_puts_errors(src,out=$stdout)
+    ret=parse(src)
+    unless ret
+      out.puts parser_failure_info
+    end
+    ret
+  end
+
+  def node_list_string(node_list,common_root=[])
+    node_list && node_list[common_root.length..-1].map{|p|"#{p.class}(#{p.offset})"}.join(" > ")
+  end
+
+  def parser_failure_info
+    return unless src
+    bracketing_lines=5
+    line,col=src.line_col(failure_index)
+    ret=<<-ENDTXT
+Parsing error at line #{line} column #{col} offset #{failure_index}
+
+Source:
+...
+#{(failure_index==0 ? "" : src[0..(failure_index-1)]).last_lines(bracketing_lines)}<HERE>#{src[(failure_index)..-1].first_lines(bracketing_lines)}
+...
+ENDTXT
+
+      if @parsing_did_not_match_entire_input
+        ret+="\nParser did not match entire input."
+      else
+
+        common_root=nil
+        expecting_list.values.each do |e|
+          node=e[:node]
+          pl=node.parent_list
+          if common_root
+            common_root.each_index do |i|
+              if pl[i]!=common_root[i]
+                common_root=common_root[0..i-1]
+                break
+              end
+            end
+          else
+            common_root=node.parent_list
+          end
+        end
+        ret+=<<ENDTXT
+
+Successfully matched rules up to failure:
+  #{node_list_string(common_root)}
+
+Expecting#{expecting_list.length>1 ? ' one of' : ''}:
+  #{expecting_list.values.collect do |a|
+    list=node_list_string(a[:node].parent_list,common_root)
+    [list,"#{a[:pattern].inspect} (#{list})"]
+  end.sort.map{|i|i[1]}.join("\n  ")}
+ENDTXT
+    end
+    ret
+  end
+end
+end

data/lib/pattern_element.rb

@@ -5,189 +5,190 @@ http://babel-bridge.rubyforge.org/
 =end
 
 module BabelBridge
-  # hash which can be used declaratively
-  class PatternElementHash < Hash
-    def method_missing(method_name, *args)  #method_name is a symbol
-      return self if args.length==1 && !args[0] # if nil is provided, don't set anything
-      self[method_name]=args[0] || true # on the other hand, if no args are provided, assume true
-      self
-    end
+# hash which can be used declaratively
+class PatternElementHash < Hash
+  def method_missing(method_name, *args)  #method_name is a symbol
+    return self if args.length==1 && !args[0] # if nil is provided, don't set anything
+    raise "More than one argument is not supported. #{self.class}##{method_name} args=#{args.inspect}" if args.length > 1
+    self[method_name]=args[0] || true # on the other hand, if no args are provided, assume true
+    self
+  end
+end
+
+# PatternElement provides optimized parsing for each Element of a pattern
+# PatternElement provides all the logic for parsing:
+#   :many
+#   :optional
+class PatternElement
+  attr_accessor :parser,:optional,:negative,:name,:terminal,:could_match
+  attr_accessor :match,:rule_variant
+
+  #match can be:
+  # true, Hash, Symbol, String, Regexp
+  def initialize(match,rule_variant)
+    self.rule_variant=rule_variant
+    init(match)
+
+    raise "pattern element cannot be both :dont and :optional" if negative && optional
   end
 
-  # PatternElement provides optimized parsing for each Element of a pattern
-  # PatternElement provides all the logic for parsing:
-  #   :many
-  #   :optional
-  class PatternElement
-    attr_accessor :parser,:optional,:negative,:name,:terminal,:could_match
-    attr_accessor :match,:rule_variant
-
-    #match can be:
-    # true, Hash, Symbol, String, Regexp
-    def initialize(match,rule_variant)
-      self.rule_variant=rule_variant
-      init(match)
-
-      raise "pattern element cannot be both :dont and :optional" if negative && optional
-    end
+  def to_s
+    match.inspect
+  end
 
-    def to_s
-      match.inspect
-    end
+  # attempt to match the pattern defined in self.parser in parent_node.src starting at offset parent_node.next
+  def parse(parent_node)
+    # run element parser
+    match=parser.call(parent_node)
 
-    # attempt to match the pattern defined in self.parser in parent_node.src starting at offset parent_node.next
-    def parse(parent_node)
-      # run element parser
-      match=parser.call(parent_node)
+    # Negative patterns (PEG: !element)
+    match=match ? nil : EmptyNode.new(parent_node) if negative
 
-      # Negative patterns (PEG: !element)
-      match=match ? nil : EmptyNode.new(parent_node) if negative
+    # Optional patterns (PEG: element?)
+    match=EmptyNode.new(parent_node) if !match && optional
 
-      # Optional patterns (PEG: element?)
-      match=EmptyNode.new(parent_node) if !match && optional
+    # Could-match patterns (PEG: &element)
+    match.match_length=0 if match && could_match
 
-      # Could-match patterns (PEG: &element)
-      match.match_length=0 if match && could_match
+    # return match
+    match
+  end
 
-      # return match
-      match
+  private
+
+  # initialize PatternElement based on the type of: match
+  def init(match)
+    self.match=match
+    case match
+    when TrueClass then init_true
+    when String then    init_string match
+    when Regexp then    init_regex match
+    when Symbol then    init_rule match
+    when Hash then      init_hash match
+    else                raise "invalid pattern type: #{match.inspect}"
     end
+  end
 
-    private
-
-    # initialize PatternElement based on the type of: match
-    def init(match)
-      self.match=match
-      case match
-      when TrueClass then init_true
-      when String then    init_string match
-      when Regexp then    init_regex match
-      when Symbol then    init_rule match
-      when Hash then      init_hash match
-      else                raise "invalid pattern type: #{match.inspect}"
-      end
-    end
+  # "true" parser always matches the empty string
+  def init_true
+    self.parser=lambda {|parent_node| EmptyNode.new(parent_node)}
+  end
 
-    # "true" parser always matches the empty string
-    def init_true
-      self.parser=lambda {|parent_node| EmptyNode.new(parent_node)}
+  # initialize PatternElement as a parser that matches exactly the string specified
+  def init_string(string)
+    self.parser=lambda do |parent_node|
+      if parent_node.src[parent_node.next,string.length]==string
+        TerminalNode.new(parent_node,string.length,string)
+      end
     end
+    self.terminal=true
+  end
 
-    # initialize PatternElement as a parser that matches exactly the string specified
-    def init_string(string)
-      self.parser=lambda do |parent_node|
-        if parent_node.src[parent_node.next,string.length]==string
-          TerminalNode.new(parent_node,string.length,string)
-        end
+  # initialize PatternElement as a parser that matches the given regex
+  def init_regex(regex)
+    optimized_regex=/\A#{regex}/  # anchor the search
+    self.parser=lambda do |parent_node|
+      offset=parent_node.next
+      if parent_node.src[offset..-1].index(optimized_regex)==0
+        range=$~.offset(0)
+        TerminalNode.new(parent_node,range[1]-range[0],regex)
       end
-      self.terminal=true
     end
+    self.terminal=true
+  end
 
-    # initialize PatternElement as a parser that matches the given regex
-    def init_regex(regex)
-      optimized_regex=/\A#{regex}/  # anchor the search
-      self.parser=lambda do |parent_node|
-        offset=parent_node.next
-        if parent_node.src[offset..-1].index(optimized_regex)==0
-          range=$~.offset(0)
-          TerminalNode.new(parent_node,range[1]-range[0],regex)
-        end
-      end
-      self.terminal=true
+  # initialize PatternElement as a parser that matches a named sub-rule
+  def init_rule(rule_name)
+    rule_name.to_s[/^([^?!]*)([?!])?$/]
+    rule_name=$1.to_sym
+    option=$2
+    match_rule=rule_variant.rule.parser.rules[rule_name]
+    raise "no rule for #{rule_name}" unless match_rule
+
+    self.parser = lambda {|parent_node| match_rule.parse(parent_node)}
+    self.name   = rule_name
+    case option
+    when "?"  then self.optional=true
+    when "!"  then self.negative=true
     end
+  end
 
-    # initialize PatternElement as a parser that matches a named sub-rule
-    def init_rule(rule_name)
-      rule_name.to_s[/^([^?!]*)([?!])?$/]
-      rule_name=$1.to_sym
-      option=$2
-      match_rule=rule_variant.rule.parser.rules[rule_name]
-      raise "no rule for #{rule_name}" unless match_rule
-
-      self.parser = lambda {|parent_node| match_rule.parse(parent_node)}
-      self.name   = rule_name
-      case option
-      when "?"  then self.optional=true
-      when "!"  then self.negative=true
-      end
+  # initialize the PatternElement from hashed parameters
+  def init_hash(hash)
+    if hash[:parser]
+      self.parser=hash[:parser]
+    elsif hash[:many]
+      init_many hash
+    elsif hash[:match]
+      init hash[:match]
+    else
+      raise "extended-options patterns (specified by a hash) must have either :parser=> or a :match=> set"
     end
 
-    # initialize the PatternElement from hashed parameters
-    def init_hash(hash)
-      if hash[:parser]
-        self.parser=hash[:parser]
-      elsif hash[:many]
-        init_many hash
-      elsif hash[:match]
-        init hash[:match]
-      else
-        raise "extended-options patterns (specified by a hash) must have either :parser=> or a :match=> set"
-      end
+    self.name = hash[:as] || self.name
+    self.optional ||= hash[:optional] || hash[:optionally]
+    self.could_match ||= hash[:could]
+    self.negative ||= hash[:dont]
+  end
+
+  # initialize the PatternElement as a many-parser from hashed parameters (hash[:many] is assumed to be set)
+  def init_many(hash)
+    # generate single_parser
+    init hash[:many]
+    single_parser=parser
+
+    # generate delimiter_pattern_element
+    delimiter_pattern_element= hash[:delimiter] && PatternElement.new(hash[:delimiter],rule_variant)
 
-      self.name = hash[:as] || self.name
-      self.optional ||= hash[:optional] || hash[:optionally]
-      self.could_match ||= hash[:could]
-      self.negative ||= hash[:dont]
+    # generate post_delimiter_element
+    post_delimiter_element=hash[:post_delimiter] && case hash[:post_delimiter]
+    when TrueClass then delimiter_pattern_element
+    else PatternElement.new(hash[:post_delimiter],rule_variant)
     end
 
-    # initialize the PatternElement as a many-parser from hashed parameters (hash[:many] is assumed to be set)
-    def init_many(hash)
-      # generate single_parser
-      init hash[:many]
-      single_parser=parser
+    # generate many-parser
+    self.parser= lambda do |parent_node|
+      last_match=single_parser.call(parent_node)
+      many_node=ManyNode.new(parent_node)
 
-      # generate delimiter_pattern_element
-      delimiter_pattern_element= hash[:delimiter] && PatternElement.new(hash[:delimiter],rule_variant)
+      if delimiter_pattern_element
+        # delimited matching
+        while last_match
+          many_node<<last_match
 
-      # generate post_delimiter_element
-      post_delimiter_element=hash[:post_delimiter] && case hash[:post_delimiter]
-      when TrueClass then delimiter_pattern_element
-      else PatternElement.new(hash[:post_delimiter],rule_variant)
-      end
+          #match delimiter
+          delimiter_match = delimiter_pattern_element.parse(many_node)
+          break unless delimiter_match
+          many_node.delimiter_matches<<delimiter_match
 
-      # generate many-parser
-      self.parser= lambda do |parent_node|
-        last_match=single_parser.call(parent_node)
-        many_node=ManyNode.new(parent_node)
-
-        if delimiter_pattern_element
-          # delimited matching
-          while last_match
-            many_node<<last_match
-
-            #match delimiter
-            delimiter_match=delimiter_pattern_element.parse(many_node)
-            break unless delimiter_match
-            many_node.delimiter_matches<<delimiter_match
-
-            #match next
-            last_match=single_parser.call(many_node)
-          end
-        else
-          # not delimited matching
-          while last_match
-            many_node<<last_match
-            last_match=single_parser.call(many_node)
-          end
+          #match next
+          last_match=single_parser.call(many_node)
         end
+      else
+        # not delimited matching
+        while last_match
+          many_node<<last_match
+          last_match=single_parser.call(many_node)
+        end
+      end
 
-        # success only if we have at least one match
-        return nil unless many_node.length>0
-
-        # pop the post delimiter matched with delimiter_pattern_element
-        many_node.delimiter_matches.pop if many_node.length==many_node.delimiter_matches.length
+      # success only if we have at least one match
+      return nil unless many_node.length>0
 
-        # If post_delimiter is requested, many_node and delimiter_matches will be the same length
-        if post_delimiter_element
-          post_delimiter_match=post_delimiter_element.parse(many_node)
+      # pop the post delimiter matched with delimiter_pattern_element
+      many_node.delimiter_matches.pop if many_node.length==many_node.delimiter_matches.length
 
-          # fail if post_delimiter didn't match
-          return nil unless post_delimiter_match
-          many_node.delimiter_matches<<post_delimiter_match
-        end
+      # If post_delimiter is requested, many_node and delimiter_matches will be the same length
+      if post_delimiter_element
+        post_delimiter_match=post_delimiter_element.parse(many_node)
 
-        many_node
+        # fail if post_delimiter didn't match
+        return nil unless post_delimiter_match
+        many_node.delimiter_matches<<post_delimiter_match
       end
+
+      many_node
     end
   end
+end
 end