babel_bridge 0.4.1 → 0.5.0

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/nodes.rb

@@ -4,7 +4,7 @@
   terminal_node
   non_terminal_node
   rule_node
-  many_node
+  root_node
 }.each do |file|
   require File.join(File.dirname(__FILE__),"nodes",file)
 end

data/lib/nodes/empty_node.rb

@@ -11,29 +11,10 @@ module BabelBridge
 # not subclassed
 class EmptyNode < Node
   def inspect(options={})
-    "EmptyNode" unless options[:simple]
+    "EmptyNode" if options[:verbose]
   end
 
   def matches; [self]; end
-
 end
 
-class RollbackWhitespaceNode < Node
-  def inspect(options={})
-    "RollbackWhitespace" unless options[:simple]
-  end
-
-  def matches; [self]; end
-
-  def initialize(parent)
-    super
-    self.match_length = 0
-    self.offset = parent.postwhitespace_range.first
-  end
-
-  def postwhitespace_range
-    @postwhitespace_range ||= offset_after_match .. offset_after_match-1
-  end
-
-end
 end

data/lib/nodes/node.rb

@@ -7,15 +7,10 @@ end
 
 # base class for all parse-tree nodes
 class Node
-  attr_accessor :src,:offset,:match_length,:parent,:parser,:prewhitespace_range
+  attr_accessor :src,:offset,:match_length,:parent,:parser,:delimiter,:many_delimiter
 
-  # no_postwhitespace is used when parsing to temporarilly rollback the preceeding whitespace while
-  # attempting to match an ignore_whitespace pattern.
-  # It should always be false again once parsing completes or fails.
-  attr_accessor :no_postwhitespace
-
-  def whitespace_regexp
-    parser.whitespace_regexp
+  def relative_class_name
+    (self.class.to_s.split(parser.class.to_s+"::",2)[1]||self.class.to_s).strip
   end
 
   # the index of the first character after the match
@@ -23,29 +18,30 @@ class Node
     offset + match_length
   end
 
-  def match_range
-    offset..(offset+match_length-1)
+  def remaining_src(sub_offset)
+    src[self.next+sub_offset..-1]
   end
 
-  def postwhitespace_range_without_no_postwhitespace
-    parser.white_space_range offset_after_match
+  def match_range
+    offset..(offset+match_length-1)
   end
 
-  def postwhitespace_range
-    r = postwhitespace_range_without_no_postwhitespace
-    no_postwhitespace ? r.first..r.first-1 : r
+  # called when a ruled is matched
+  def on_matched
   end
 
-  def postwhitespace
-    src[postwhitespace_range]
+  def init_line_column
+    @line, @column = Tools.line_column(src, offset)
   end
 
-  def prewhitespace
-    src[prewhitespace_range]
+  def line
+    init_line_column unless @line
+    @line
   end
 
-  # called when a ruled is matched
-  def matched
+  def column
+    init_line_column unless @column
+    @column
   end
 
   def to_s
@@ -67,7 +63,6 @@ class Node
       self.parent=parent_or_parser
       self.parser=parent.parser
       self.offset=parent.next
-      self.prewhitespace_range=parent.postwhitespace_range
       self.src=parent.src
       raise "parent node does not have parser set" unless parser
     else
@@ -79,12 +74,6 @@ class Node
     node_init(parent)
   end
 
-  # after a node has been matched, the node will get this called on itself
-  # It can then rewrite itself however it wishes
-  def post_match
-    self
-  end
-
   # Returns a human-readable representation of the parse tree
   # options
   #   :simple => output a simplified representation of the parse tree
@@ -95,7 +84,7 @@ class Node
   #********************
   # info methods
   #********************
-  def next; postwhitespace_range.last+1 end # index of first character after match and any trailing whitespace
+  alias :next :offset_after_match
   def text; src[match_range] end  # the substring in src matched
 
   # length returns the number of sub-nodes
@@ -107,11 +96,23 @@ class Node
     return parent ? parent.parent_list+[parent] : []
   end
 
+  # walk down the children chain as long as there is only one child at each level
+  # log and return the path
+  def onlychildren_list
+    if matches.length == 1
+      [self] + matches[0].onlychildren_list
+    else
+      [self]
+    end
+  end
+
+  def path_string(node_list)
+    node_list.collect{|n|n.class}.join ' > '
+  end
+
   def node_path
-    "#{parent && (parent.node_path+' > ')}#{self.class}(#{offset})"
+    path_string parent_list
   end
 end
 
-class RootNode < Node
-end
 end

data/lib/nodes/non_terminal_node.rb

@@ -8,18 +8,9 @@ module BabelBridge
 # rule node
 # subclassed automatically by parser.rule for each unique non-terminal
 class NonTerminalNode < Node
-  attr_accessor :last_non_empty_node
-
-  def postwhitespace_range_without_no_postwhitespace
-    if  last_non_empty_node
-      last_non_empty_node.postwhitespace_range
-    else
-      prewhitespace_range || (0..-1)
-    end
-  end
 
   def update_match_length
-    @match_length = last_non_empty_node ? last_non_empty_node.offset_after_match - offset : 0
+    @match_length = last_match ? last_match.offset_after_match - offset : 0
   end
 
   #*****************************
@@ -29,15 +20,21 @@ class NonTerminalNode < Node
     @matches ||= []
   end
 
+  def last_match
+    matches[-1]
+  end
+
   include Enumerable
   def length
     matches.length
   end
 
   def add_match(node)
-    @last_non_empty_node = node unless node.kind_of?(EmptyNode)
-    matches<<node
-    update_match_length
+    return if !node || node.kind_of?(EmptyNode) || node == self
+    node.tap do
+      matches << node
+      update_match_length
+    end
   end
 
   def [](i)

data/lib/nodes/root_node.rb

@@ -0,0 +1,4 @@
+module BabelBridge
+class RootNode < Node
+end
+end

data/lib/nodes/rule_node.rb

@@ -9,40 +9,50 @@ module BabelBridge
 # subclassed automatically by parser.rule for each unique non-terminal
 class RuleNode < NonTerminalNode
 
+  def initialize(parent, delimiter_pattern = nil)
+    @num_match_attempts = 0
+    @delimiter_pattern = delimiter_pattern
+    super parent
+  end
+
   def match_names
     @match_names ||= []
   end
 
-  def matches_by_name
-    @matches_by_name||= begin
-      raise "matches.length #{matches.length} != match_names.length #{match_names.length}" unless matches.length==match_names.length
-      mbn={}
-      mn=match_names
-      matches.each_with_index do |match,i|
-        name=mn[i]
-        next unless name
-        if current=mbn[name] # name already used
-          # convert to MultiMatchesArray if not already
-          mbn[name]=MultiMatchesArray.new([current]) if !current.kind_of? MultiMatchesArray
-          # add to array
-          mbn[name]<<match
-        else
-          mbn[name]=match
-        end
-      end
-      mbn
+  def match_name_is_poly(name)
+    return unless name
+    if current = matches_by_name[name]
+      matches_by_name[name] = MultiMatchesArray.new([current]) if !current.kind_of? MultiMatchesArray
+    else
+      matches_by_name[name] = MultiMatchesArray.new
     end
   end
 
+  def add_match_name(match,name)
+    return unless name
+    if current = matches_by_name[name]
+      matches_by_name[name] = MultiMatchesArray.new([current]) if !current.kind_of? MultiMatchesArray
+      matches_by_name[name] << match
+    else
+      matches_by_name[name] = match
+    end
+  end
+
+  def matches_by_name
+    @matches_by_name||={}
+  end
+
   def inspect(options={})
-    return "#{self.class}" if matches.length==0
-    matches_inspected=matches.collect{|a|a.inspect(options)}.compact
+    return relative_class_name if matches.length==0
+    matches = @matches
+    matches=matches.select{|m|!m.many_delimiter} unless options[:verbose]
+    matches_inspected = matches.collect{|a|a.inspect(options)}.compact
     if matches_inspected.length==0 then nil
     elsif matches_inspected.length==1
-      m=matches_inspected[0]
-      ret="#{self.class} > "+matches_inspected[0]
+      m = matches_inspected[0]
+      ret = "#{relative_class_name} > "+matches_inspected[0]
       if options[:simple]
-        ret=if m["\n"] then m
+        ret = if m["\n"] then m
         else
           # just show the first and last nodes in the chain
           ret.gsub(/( > [A-Z][a-zA-Z0-9:]+ > (\.\.\. > )?)/," > ... > ")
@@ -50,18 +60,17 @@ class RuleNode < NonTerminalNode
       end
       ret
     else
-      (["#{self.class}"]+matches_inspected).join("\n").gsub("\n","\n  ")
+      (["#{relative_class_name}"]+matches_inspected).join("\n").gsub("\n","\n  ")
     end
   end
 
   #********************
   # alter methods
   #********************
-  def reset_matches_by_name
-    @matches_by_name=nil
-  end
 
-  # defines where to forward missing methods to; override for custom behavior
+  # returns where to forward missing methods calls to (safe to override for custom behavior; respond_to? and method_missing will "do the right thing")
+  # returns nil if there is no object to forward to that will respond to the call
+  # default: forward to the first match that responds to method_name
   def forward_to(method_name)
     matches.each {|m| return m if m.respond_to?(method_name)}
     nil
@@ -74,29 +83,73 @@ class RuleNode < NonTerminalNode
   end
 
   def method_missing(method_name, *args)  #method_name is a symbol
-    unless matches_by_name.has_key? method_name
-      if f=forward_to(method_name)
-        return f.send(method_name,*args)
-      end
-      match_path = [self]
-      while match_path[-1].matches.length==1
-        match_path<<match_path[-1].matches[0]
-      end
-      raise "#{match_path.collect{|m|m.class}.join(' > ')}: no methods or named pattern elements match: #{method_name.inspect}"
-    end
-    case ret=matches_by_name[method_name]
-    when EmptyNode then nil
-    else ret
+    return matches_by_name[method_name] if matches_by_name.has_key?(method_name)
+
+    if f = forward_to(method_name)
+      return f.send(method_name,*args)
     end
+
+    raise "#{path_string onlychildren_list}: no methods or named pattern elements match: #{method_name.inspect} on #{self.class} instance"
   end
 
   # adds a match with name (optional)
   def add_match(match,name=nil)
-    reset_matches_by_name
-    super match
-    match_names<<name
+    return unless match
+    return match if match==self
+
+    add_match_name(super(match),name)
 
     update_match_length
+    match
+  end
+
+  def pop_match
+    matches.pop.tap {update_match_length}
+  end
+
+  # Attempts to match the pattern_element starting at the end of what has already been matched
+  # If successful, adds the resulting Node to matches.
+  # returns nil on if pattern_element wasn't matched; non-nil if it was skipped or matched
+  def match(pattern_element)
+    @num_match_attempts += 1
+    return :no_pattern_element unless pattern_element
+    return :skipped if pattern_element.delimiter &&
+      (
+      if last_match
+        last_match.delimiter       # don't match two delimiters in a row
+      else
+        @num_match_attempts > 1   # don't match a delimiter as the first element unless this is the first match attempt
+      end
+      )
+
+    if result = pattern_element.parse(self)
+      add_match result, pattern_element.name # success, but don't keep EmptyNodes
+    end
   end
+
+  def match_delimiter
+    match @delimiter_pattern
+  end
+
+  # called after matching is done and it was a success
+  # returns the node which is actually added to the parse tree
+  def post_match_processing
+    on_matched
+    self
+  end
+
+  # a simple "transaction" - logs the curent number of matches,
+  # if the block's result is false, it discards all new matches
+  def attempt_match
+    matches_before = matches.length
+    match_length_before = match_length
+    (yield && match_length > match_length_before).tap do |success|  # match_length test returns failure if no progress is made (our source position isn't advanced)
+      unless success
+        @matches = matches[0..matches_before-1]
+        update_match_length
+      end
+    end
+  end
+
 end
 end

data/lib/nodes/terminal_node.rb

@@ -8,7 +8,7 @@ module BabelBridge
 # used for String and Regexp PatternElements
 # not subclassed
 class TerminalNode < Node
-  attr_accessor :pattern, :postwhitespace_offset
+  attr_accessor :pattern
   def initialize(parent,range,pattern)
     node_init(parent)
     self.offset = range.min
@@ -17,10 +17,10 @@ class TerminalNode < Node
   end
 
   def inspect(options={})
-    "#{text.inspect}" unless options[:simple] && text[/^\s*$/] # if simple && node only matched white-space, return nil
+    "#{text.inspect}" unless !options[:verbose] && text[/^\s*$/] # if only show whitespace matches if verbose
   end
 
-  def matches; [self]; end
+  def matches; []; end
 
 end
 end

data/lib/parser.rb

@@ -8,7 +8,7 @@ class Parser
   # These methods are used in the creation of a Parser Sub-Class to define
   # its grammar
   class <<self
-    attr_accessor :rules, :module_name, :root_rule, :whitespace_regexp
+    attr_accessor :rules, :module_name, :root_rule, :delimiter_pattern
 
     def rules
       @rules||={}
@@ -37,19 +37,20 @@ class Parser
     #
     # The block is executed in the context of the rule-varient's node type, a subclass of: RuleNode
     # This allows you to add whatever functionality you want to a your nodes in the final parse tree.
-    # Also note you can override the post_match method. This allows you to restructure the parse tree as it is parsed.
+    # Also note you can override the on_post_match method. This allows you to restructure the parse tree as it is parsed.
     def rule(name,*pattern,&block)
       rule = self.rules[name] ||= Rule.new(name,self)
-      self.root_rule ||= name
-      rule.add_variant(pattern,&block)
+      @root_rule ||= name
+      options =  pattern[-1].kind_of?(Hash) ? pattern.pop : {}
+      rule.add_variant options.merge(:pattern => pattern), &block
     end
 
     # options
     # => right_operators: list of all operators that should be evaluated right to left instead of left-to-write
     #       typical example is the "**" exponentiation operator which should be evaluated right-to-left.
-    def binary_operators_rule(name,elements_pattern,operators,options={},&block)
+    def binary_operators_rule(name,operand_rule_name,operators,options={},&block)
       right_operators = options[:right_operators]
-      rule(name,many(elements_pattern,Tools::array_to_or_regexp(operators))) do
+      rule name, many(operand_rule_name,Tools::array_to_or_regexp(operators)).delimiter_name(:operators).as(:operands) do
         self.class_eval &block if block
         class <<self
           attr_accessor :operators_from_rule, :right_operators
@@ -64,16 +65,17 @@ class Parser
           @operator||=operator_node.to_s.to_sym
         end
 
-        # Override the post_match method to take the results of the "many" match
+        def operator_processor
+          self.class.operator_processor
+        end
+
+        # Override the on_post_match method to take the results of the "many" match
         # and restructure it into a binary tree of nodes based on the precidence of
         # the "operators".
-        # TODO - I think maybe post_match should be run after the whole tree matches. If not, will this screw up caching?
-        def post_match
-          many_match = matches[0]
-          operands = many_match.matches
-          operators = many_match.delimiter_matches
-          # TODO - now! take many_match.matches and many_match.delimiter_matches, mishy-mashy, and make the super-tree!
-          self.class.operator_processor.generate_tree operands, operators, parent
+        # TODO - Should on_post_match be run after the whole tree matches? If not, will this screw up caching?
+        def post_match_processing
+          super
+          operator_processor.generate_tree operands, operators, parent
         end
       end
     end
@@ -95,13 +97,25 @@ class Parser
       @root_rule=rule
     end
 
-    def ignore_whitespace(regexp = /\s*/)
-      @whitespace_regexp = /\A(#{regexp})?/
+    def ignore_whitespace
+      delimiter /\s*/
+    end
+
+    def delimiter(*pattern)
+      @delimiter = pattern
+    end
+
+    def delimiter_pattern
+      @delimiter_pattern ||= @delimiter && PatternElement.new(@delimiter, :parser_class => self, :delimiter => true)
     end
   end
 
-  def whitespace_regexp
-    self.class.whitespace_regexp || /\A/
+  def delimiter_pattern
+    self.class.delimiter_pattern
+  end
+
+  def rules
+    self.class.rules
   end
 
   #*********************************************
@@ -137,11 +151,10 @@ class Parser
     def match(*args) PatternElementHash.new.match(*args) end
     def match!(*args) PatternElementHash.new.dont.match(*args) end
 
-    def rewind_whitespace; PatternElementHash.new.rewind_whitespace end
-
     def dont; PatternElementHash.new.dont end
     def optionally; PatternElementHash.new.optionally end
     def could; PatternElementHash.new.could end
+    def custom_parser(&block); PatternElementHash.new.parser(lambda &block) end
   end
 
 
@@ -162,24 +175,12 @@ class Parser
   end
 
   def reset_parser_tracking
+    @matching_negative_depth = 0
     @parsing_did_not_match_entire_input = false
     @src = nil
     @failure_index = 0
     @expecting_list = {}
     @parse_cache = {}
-    @white_space_ranges = {}
-  end
-
-  # memoizing whitespace parser
-  def white_space_range(start)
-    @white_space_ranges[start]||=begin
-      # src should always be a string - unless this is called AFTER parsing is done. Currently this can happen with the way ManyNode handles .match_length and .next
-      # We should be able to just use:
-      #   src[start..-1].index whitespace_regexp
-      ((src||"")[start..-1]||"").index whitespace_regexp
-      r = $~.offset 0
-      start+r[0] .. start+r[1]-1
-    end
   end
 
   def cached(rule_class,offset)
@@ -195,7 +196,9 @@ class Parser
   end
 
   def log_parsing_failure(index,expecting)
-    if index>failure_index
+    if matching_negative?
+      # ignored
+    elsif index>failure_index
       @expecting_list = {expecting[:pattern] => expecting}
       @failure_index = index
     elsif index == failure_index
@@ -205,25 +208,45 @@ class Parser
     end
   end
 
-  def parse(src,offset=0,rule=nil)
+  def matching_negative
+    @matching_negative_depth||=0
+    @matching_negative_depth+=1
+  end
+
+  def unmatching_negative
+    @matching_negative_depth-=1
+  end
+
+  def matching_negative?
+    (@matching_negative_depth||0) > 0
+  end
+
+  # parse a string, return the root node of the parse tree.
+  # If nil is returned, parsing failed. Call .parser_failure_info after failure for a human-readable description of the failure.
+  # src: the string to parse
+  # options:
+  #   offset: where to start in the string for parsing
+  #   rule: lets you specify the root rule for matching
+  #   partial_match: allow partial matching
+  def parse(src, options={})
+    offset = options[:offset] || 0
+    rule = options[:rule] || self.class.root_rule
     reset_parser_tracking
-    @start_time=Time.now
-    self.src=src
-    root_node=RootNode.new(self)
-    raise "No root rule defined." unless rule || self.class.root_rule
-    ret=self.class[rule||self.class.root_rule].parse(root_node)
-    unless rule
-      if ret
-        if ret.next<src.length # parse only succeeds if the whole input is matched
-          if ret.next >= @failure_index
-            @parsing_did_not_match_entire_input=true
-            @failure_index = ret.next
-            @failed_parse = ret
-          end
-          ret=nil
-        else
-          reset_parser_tracking
+    @start_time = Time.now
+    self.src = src
+    root_node = RootNode.new(self)
+    raise "No root rule defined." unless rule
+    ret = rules[rule].parse(root_node)
+    if ret
+      if ret.next<src.length && !options[:partial_match] # parse only succeeds if the whole input is matched
+        if ret.next >= @failure_index
+          @parsing_did_not_match_entire_input=true
+          @failure_index = ret.next
+          @failed_parse = ret
         end
+        ret=nil
+      else
+        reset_parser_tracking
       end
     end
     @end_time=Time.now
@@ -242,8 +265,18 @@ class Parser
     ret
   end
 
-  def node_list_string(node_list,common_root=[])
-    node_list && node_list[common_root.length..-1].map{|p|"#{p.class}(#{p.offset})"}.join(" > ")
+  # options[:verbose] => false
+  def node_list_string(node_list,common_root=[],options={})
+    return unless node_list
+    if options[:verbose]
+      node_list[common_root.length..-1]
+    else
+      [node_list[-1]]
+    end.select do |p|
+      p.class.to_s.index(BabelBridge.to_s)!=0
+    end.map do |p|
+      "#{p.relative_class_name}"
+    end.join(" > ")
   end
 
   def nodes_interesting_parse_path(node)
@@ -254,7 +287,8 @@ class Parser
   end
 
 
-  def expecting_output
+  # options[:verbose] => false
+  def expecting_output(options={})
     return "" if expecting_list.length==0
     common_root=nil
     expecting_list.values.each do |e|
@@ -275,13 +309,13 @@ class Parser
     <<ENDTXT
 
 Parse path at failure:
-  #{node_list_string(common_root)}
+  #{node_list_string(common_root,[],:verbose=>true)}
 
 Expecting#{expecting_list.length>1 ? ' one of' : ''}:
-  #{expecting_list.values.collect do |a|
-    list=node_list_string(nodes_interesting_parse_path(a[:node]),common_root)
-    [list,"#{a[:pattern].inspect} (#{list})"]
-  end.sort.map{|i|i[1]}.join("\n  ")}
+#{Tools.uniform_tabs(Tools.indent(expecting_list.values.collect do |a|
+    list=node_list_string(nodes_interesting_parse_path(a[:node]),common_root,options)
+    "#{a[:pattern].inspect}\t#{list}"
+  end.sort.join("\n"),"  "))}
 ENDTXT
   end
 
@@ -290,7 +324,7 @@ ENDTXT
     return unless src
     verbose = options[:verbose]
     bracketing_lines=5
-    line,col=src.line_col(failure_index)
+    line,col = Tools.line_column(src, failure_index)
     ret=<<-ENDTXT
 Parsing error at line #{line} column #{col} offset #{failure_index}
 
@@ -303,7 +337,7 @@ ENDTXT
     if @parsing_did_not_match_entire_input
       ret+="\nParser did not match entire input.\n"
       if verbose
-        ret+="\nParsed:\n#{Tools::indent failed_parse.inspect}\n"
+        ret+="\nParsed:\n#{Tools::indent failed_parse.inspect||"(nothing)"}\n"
       end
     end