treetop 1.5.3 → 1.6.12

data/lib/treetop/compiler/metagrammar.treetop

@@ -2,15 +2,15 @@ module Treetop
   module Compiler
     grammar Metagrammar
       rule treetop_file
-        requires:(space? require_statement)* prefix:space? module_or_grammar suffix:space? {
+        requires:(space? require_statement)* spacing:space? module_or_grammar suffix:space? {
           def compile
-            requires.text_value + prefix.text_value + module_or_grammar.compile + suffix.text_value
+            requires.text_value + spacing.text_value + module_or_grammar.compile + suffix.text_value
           end
         }
       end
 
       rule require_statement
-        prefix:space? "require" [ \t]+ [^\n\r]+ [\n\r]
+        spacing:space? "require" [ \t]+ [^\n\r]+ [\n\r]
       end
 
       rule module_or_grammar
@@ -18,13 +18,13 @@ module Treetop
       end
 
       rule module_declaration
-        prefix:('module' space name:([A-Z] alphanumeric_char* ('::' [A-Z] alphanumeric_char*)*) space) module_contents:(module_declaration / grammar) suffix:(space 'end') {
+        module_prefix:(('module'/'class') space name:([A-Z] alphanumeric_char* ('::' [A-Z] alphanumeric_char*)*) space) module_contents:(module_declaration / grammar) suffix:(space 'end') {
           def compile
-            prefix.text_value + module_contents.compile + suffix.text_value
+            module_prefix.text_value + module_contents.compile + suffix.text_value
           end
 
 	  def parser_name
-	    prefix.name.text_value+'::'+module_contents.parser_name
+	    module_prefix.name.text_value+'::'+module_contents.parser_name
 	  end
         }
       end
@@ -84,9 +84,17 @@ module Treetop
             super.elements.map {|elt| elt.alternative}
           end
 
+	  def parent_modules
+	    []
+	  end
+
           def inline_modules
             (alternatives.map {|alt| alt.inline_modules }).flatten
           end
+
+          def inline_module
+	    nil
+	  end
         }
       end
 
@@ -100,12 +108,20 @@ module Treetop
             sequence_body.tail
           end
 
+	  def parent_modules
+	    node_class_declarations.inline_modules
+	  end
+
           def inline_modules
             (sequence_elements.map {|elt| elt.inline_modules}).flatten +
             [sequence_element_accessor_module] +
-            node_class_declarations.inline_modules
+            parent_modules
           end
 
+          def inline_module
+	    node_class_declarations.inline_module
+	  end
+
           def inline_module_name
             node_class_declarations.inline_module_name
           end
@@ -150,29 +166,46 @@ module Treetop
             atomic
           end
 
+	  def parent_modules
+	    []
+	  end
+
           def inline_modules
             atomic.inline_modules
           end
 
+          def inline_module
+	    atomic.inline_module
+	  end
+
           def inline_module_name
             nil
           end
         }
         /
-	prefix space? predicate_block {
+	prefix space? atomic:predicate_block {
           def compile(address, builder, parent_expression=nil)
             prefix.compile(address, builder, self)
           end
           def prefixed_expression
-            predicate_block
+            atomic
           end
+	  def parent_modules
+	    []
+	  end
           def inline_modules
             []
           end
+          def inline_module
+	    nil
+	  end
         }
         /
         atomic suffix node_class_declarations {
           def compile(address, builder, parent_expression=nil)
+	    if node_class_declarations.inline_module && atomic.inline_module
+	      STDERR.puts "Extraneous module ignored after suffix: #{input[interval].inspect}"
+	    end
             suffix.compile(address, builder, self)
           end
 
@@ -184,10 +217,18 @@ module Treetop
             node_class_declarations.node_class_name
           end
 
+	  def parent_modules
+	    node_class_declarations.inline_modules
+	  end
+
           def inline_modules
-            atomic.inline_modules + node_class_declarations.inline_modules
+            atomic.inline_modules + parent_modules
           end
 
+          def inline_module
+            node_class_declarations.inline_module
+	  end
+
           def inline_module_name
             node_class_declarations.inline_module_name
           end
@@ -195,6 +236,9 @@ module Treetop
         /
         atomic node_class_declarations {
           def compile(address, builder, parent_expression=nil)
+	    if node_class_declarations.inline_module && atomic.inline_module
+	      STDERR.puts "Extraneous module ignored with nested atomic: #{input[interval].inspect}"
+	    end
             atomic.compile(address, builder, self)
           end
 
@@ -202,10 +246,18 @@ module Treetop
             node_class_declarations.node_class_name
           end
 
+	  def parent_modules
+	    node_class_declarations.inline_modules
+	  end
+
           def inline_modules
-            atomic.inline_modules + node_class_declarations.inline_modules
+            atomic.inline_modules + parent_modules
           end
 
+          def inline_module
+            node_class_declarations.inline_module
+	  end
+
           def inline_module_name
             node_class_declarations.inline_module_name
           end
@@ -222,6 +274,10 @@ module Treetop
             sequence_primary.compile(lexical_address, builder)
           end
 
+	  def parent_modules
+	    []
+	  end
+
           def inline_modules
             sequence_primary.inline_modules
           end
@@ -238,6 +294,10 @@ module Treetop
             sequence_primary.compile(lexical_address, builder)
           end
 
+	  def parent_modules
+	    []
+	  end
+
           def inline_modules
             sequence_primary.inline_modules
           end
@@ -282,6 +342,10 @@ module Treetop
             elements[1]
           end
 
+	  def parent_modules
+	    []
+	  end
+
           def inline_modules
             atomic.inline_modules
           end
@@ -291,13 +355,16 @@ module Treetop
           end
         }
         /
-        prefix space? predicate_block {
+        prefix space? atomic:predicate_block {
           def compile(address, builder, parent_expression=nil)
             prefix.compile(address, builder, self)
           end
           def prefixed_expression
-            predicate_block
+            atomic
           end
+	  def parent_modules
+	    []
+	  end
           def inline_modules
             []
           end
@@ -312,6 +379,10 @@ module Treetop
             nil
           end
 
+	  def parent_modules
+	    []
+	  end
+
           def inline_modules
             atomic.inline_modules
           end
@@ -374,9 +445,15 @@ module Treetop
 
       rule parenthesized_expression
         '(' space? parsing_expression space? ')' <ParenthesizedExpression> {
+	  def parent_modules
+	    []
+	  end
           def inline_modules
             parsing_expression.inline_modules
           end
+          def inline_module
+            parsing_expression.inline_module
+          end
         }
       end
 
@@ -439,6 +516,9 @@ module Treetop
 
       rule trailing_inline_module
         space inline_module {
+	  def parent_modules
+	    []
+	  end
           def inline_modules
             [inline_module]
           end
@@ -449,6 +529,9 @@ module Treetop
         }
         /
         '' {
+	  def parent_modules
+	    []
+	  end
           def inline_modules
             []
           end

data/lib/treetop/compiler/node_classes/anything_symbol.rb

@@ -5,8 +5,8 @@ module Treetop
         super
         builder.if__ "index < input_length" do
           if address == 0 || decorated?
-	    assign_result "instantiate_node(#{node_class_name},input, index...(index + 1))"
-	    extend_result_with_inline_module
+            assign_result "instantiate_node(#{node_class_name},input, index...(index + 1))"
+            extend_result_with_inline_module parent_expression
           else
             assign_lazily_instantiated_node
           end
@@ -17,6 +17,14 @@ module Treetop
           assign_result 'nil'
         end
       end
+
+      def expected
+        '"any character"'
+      end
+
+      def inline_module
+        nil
+      end
     end
   end
 end

data/lib/treetop/compiler/node_classes/atomic_expression.rb

@@ -6,8 +6,8 @@ module Treetop
       end
       
       def single_quote(string)
-	# Double any backslashes, then backslash any single-quotes:
-	"'#{string.gsub(/\\/) { '\\\\' }.gsub(/'/) { "\\'"}}'"
+        # Double any backslashes, then backslash any single-quotes:
+        "'#{string.gsub(/\\/) { '\\\\' }.gsub(/'/) { "\\'"}}'"
       end
     end
   end

data/lib/treetop/compiler/node_classes/character_class.rb

@@ -7,18 +7,26 @@ module Treetop
         builder.if__ "has_terminal?(@regexps[gr = #{grounded_regexp(text_value)}] ||= Regexp.new(gr), :regexp, index)" do
           if address == 0 || decorated?
             assign_result "instantiate_node(#{node_class_name},input, index...(index + 1))"
-            extend_result_with_inline_module
+            extend_result_with_inline_module parent_expression
           else
             assign_lazily_instantiated_node
           end
           builder << "@index += 1"  # Always one character
         end
         builder.else_ do
-          builder << "terminal_parse_failure(#{single_quote('['+characters+']')})"
+          builder << "terminal_parse_failure(#{expected})"
           assign_result 'nil'
         end
       end
 
+      def expected
+        single_quote('['+characters+']')
+      end
+
+      def inline_module
+        nil
+      end
+
       def grounded_regexp(string)
         # Double any backslashes, then backslash any single-quotes:
         "'\\A#{string.gsub(/\\/) { '\\\\' }.gsub(/'/) { "\\'"}}'"

data/lib/treetop/compiler/node_classes/choice.rb

@@ -5,29 +5,33 @@ module Treetop
         super
         begin_comment(self)
         use_vars :result, :start_index
-        compile_alternatives(alternatives)
+        compile_alternatives(alternatives, parent_expression)
         end_comment(self)
       end
-      
-      def compile_alternatives(alternatives)
+
+      def compile_alternatives(alternatives, parent_expression)
         obtain_new_subexpression_address
         alternatives.first.compile(subexpression_address, builder)
         builder.if__ subexpression_success? do
-	  # Undo lazy instantiation:
-	  builder << "#{subexpression_result_var} = SyntaxNode.new(input, (index-1)...index) if #{subexpression_result_var} == true"
+          # Undo lazy instantiation:
+          builder << "#{subexpression_result_var} = SyntaxNode.new(input, (index-1)...index) if #{subexpression_result_var} == true"
           assign_result subexpression_result_var
           extend_result_with_declared_module
-          extend_result_with_inline_module
+          extend_result_with_inline_module parent_expression
         end
         builder.else_ do
           if alternatives.size == 1
             reset_index
             assign_failure start_index_var
           else
-            compile_alternatives(alternatives[1..-1])
+            compile_alternatives(alternatives[1..-1], parent_expression)
           end
         end
       end
+
+      def expected
+        '"(any alternative)"'
+      end
     end
   end
 end

data/lib/treetop/compiler/node_classes/nonterminal.rb

@@ -6,8 +6,12 @@ module Treetop
         use_vars :result
         assign_result text_value == 'super' ? 'super' : "_nt_#{text_value}"
         extend_result_with_declared_module
-        extend_result_with_inline_module
+        extend_result_with_inline_module parent_expression
+      end
+
+      def inline_module
+        nil
       end
     end
   end
-end
+end

data/lib/treetop/compiler/node_classes/parenthesized_expression.rb

@@ -4,6 +4,10 @@ module Treetop
       def compile(address, builder, parent_expression = nil)
         elements[2].compile(address, builder, parent_expression)
       end
+
+      def expected
+        elements[2].expected
+      end
     end
   end
-end
+end

data/lib/treetop/compiler/node_classes/parsing_expression.rb

@@ -78,7 +78,12 @@ module Treetop
         extend_result declared_module_name if declared_module_name
       end
       
-      def extend_result_with_inline_module
+      def extend_result_with_inline_module parent_expression = nil
+        if parent_expression && parent_expression.parent_modules.size > 0
+          parent_expression.parent_modules.each do |inline|
+            extend_result inline.module_name
+          end
+        end
         extend_result inline_module_name if inline_module_name
       end
     
@@ -141,6 +146,10 @@ module Treetop
       def on_one_line(expression)
         expression.text_value.tr("\n", ' ')
       end
+
+      def expected
+        nil # Overridden for terminal parse failures
+      end
     end
   end
 end

data/lib/treetop/compiler/node_classes/parsing_rule.rb

@@ -33,7 +33,7 @@ module Treetop
       
       def generate_cache_lookup(builder)
         builder.if_ "node_cache[:#{name}].has_key?(index)" do
-	  cache_address = "node_cache[:#{name}][index]"
+          cache_address = "node_cache[:#{name}][index]"
           builder.assign 'cached', cache_address
           builder.if_ "cached" do
             # Handle lazily instantiated nodes:

data/lib/treetop/compiler/node_classes/predicate.rb

@@ -13,6 +13,7 @@ module Treetop
       end
       
       def assign_failure
+        reset_index
         super(start_index_var)
       end
       
@@ -35,11 +36,17 @@ module Treetop
     class NotPredicate < Predicate
       def when_success
         assign_failure
+        if (e = parent.atomic.expected)
+          builder << "terminal_parse_failure(#{e}, true)"
+        end
       end
       
       def when_failure
+        if (e = parent.atomic.expected)
+          builder << "@terminal_failures.pop"
+        end
         assign_success
       end
     end
   end
-end
+end

data/lib/treetop/compiler/node_classes/predicate_block.rb

@@ -10,6 +10,13 @@ module Treetop
         p = parent
         p = p.parent while p && !p.respond_to?(:accumulator_var)
         assign_result "lambda #{text_value}.call(#{p ? p.accumulator_var : ""})"
+        builder.if_ '!'+result_var do
+          builder << "terminal_parse_failure(#{expected})"
+        end
+      end
+
+      def expected
+        '"<semantic predicate>"'  # Should I include (some of) the text_value here?
       end
     end
   end

data/lib/treetop/compiler/node_classes/repetition.rb

@@ -28,9 +28,9 @@ module Treetop
         parent_expression.inline_module_name
       end
 
-      def assign_and_extend_result
+      def assign_and_extend_result parent_expression
         assign_result "instantiate_node(#{node_class_name},input, #{start_index_var}...index, #{accumulator_var})"
-        extend_result_with_inline_module
+        extend_result_with_inline_module parent_expression
       end
     end
 
@@ -38,7 +38,7 @@ module Treetop
     class ZeroOrMore < Repetition
       def compile(address, builder, parent_expression)
         super
-        assign_and_extend_result
+        assign_and_extend_result parent_expression
         end_comment(parent_expression)
       end
 
@@ -55,7 +55,7 @@ module Treetop
           assign_failure start_index_var
         end
         builder.else_ do
-          assign_and_extend_result
+          assign_and_extend_result parent_expression
         end
         end_comment(parent_expression)
       end
@@ -63,26 +63,46 @@ module Treetop
       def max
         nil
       end
+
+      def expected
+        parent_expression.atomic.expected && '"at least one "+'+parent_expression.atomic.expected
+      end
     end
 
     class OccurrenceRange < Repetition
       def compile(address, builder, parent_expression)
         super
 
-        if min.empty? || min.text_value.to_i == 0
-          assign_and_extend_result
-        else
+        if !min.empty? && min.text_value.to_i != 0
           # We got some, but fewer than we wanted. There'll be a failure reported already
           builder.if__ "#{accumulator_var}.size < #{min.text_value}" do
             reset_index
             assign_failure start_index_var
           end
           builder.else_ do
-            assign_and_extend_result
+            clean_unsaturated
+            assign_and_extend_result parent_expression
           end
+        else
+          clean_unsaturated
+          assign_and_extend_result parent_expression
         end
+
         end_comment(parent_expression)
       end
+
+      # remove the last terminal_failure if we merely failed to reach the maximum
+      def clean_unsaturated
+        if !max.empty? && max.text_value.to_i > 0
+          builder.if_ "#{accumulator_var}.size < #{max.text_value}" do
+            builder << '@terminal_failures.pop'  # Ignore the last failure.
+          end
+        end
+      end
+
+      def expected
+        parent_expression.atomic.expected && "at least #{min.text_value} "+parent_expression.atomic.expected
+      end
     end
 
   end

data/lib/treetop/compiler/node_classes/sequence.rb

@@ -9,7 +9,7 @@ module Treetop
         builder.if__ "#{accumulator_var}.last" do
           assign_result "instantiate_node(#{node_class_name},input, #{start_index_var}...index, #{accumulator_var})"
           extend_result sequence_element_accessor_module_name if sequence_element_accessor_module_name
-          extend_result_with_inline_module
+          extend_result_with_inline_module parent_expression
         end
         builder.else_ do
           reset_index
@@ -40,6 +40,10 @@ module Treetop
       def sequence_element_accessor_module_name
         sequence_element_accessor_module.module_name
       end
+
+      def expected
+        '"<a sequence>"'
+      end
     end
     
     class SequenceElementAccessorModule

data/lib/treetop/compiler/node_classes/terminal.rb

@@ -3,39 +3,53 @@ module Treetop
     class Terminal < AtomicExpression
       def compile(address, builder, parent_expression = nil)
         super
-	# Handle modifiers:
-	insensitive = modifiers.text_value.include? 'i'
-	re = modifiers.text_value.include? 'r'
-	if re
-	  grounded_regexp = "#{('\A'+eval(string)).inspect}"
-	  cache_key = "'__#{modifiers.text_value}__'+(gr = #{grounded_regexp})"
-	  re_modifiers = "#{insensitive ? 'Regexp::IGNORECASE' : 0}"
-	  str = "@regexps[#{cache_key}] ||= Regexp.new(gr, #{re_modifiers})"
-	  mode = ':regexp'
-	elsif insensitive
-	  str = string.downcase
-	  string_length = eval(str).length
-	  mode = ':insens'
-	else
-	  str = string
-	  string_length = eval(str).length
-	  mode = 'false'
-	end
+        # Handle modifiers:
+        insensitive = modifiers.text_value.include? 'i'
+        re = modifiers.text_value.include? 'r'
+        if re
+          grounded_regexp = "#{('\A'+eval(string)).inspect}"
+          cache_key = "'__#{modifiers.text_value}__'+(gr = #{grounded_regexp})"
+          re_modifiers = "#{insensitive ? 'Regexp::IGNORECASE' : 0}"
+          str = "@regexps[#{cache_key}] ||= Regexp.new(gr, #{re_modifiers})"
+          mode = ':regexp'
+        elsif insensitive
+          str = string.downcase
+          string_length = eval(str).length
+          mode = ':insens'
+        else
+          str = string
+          string_length = eval(str).length
+          mode = 'false'
+        end
 
         builder.if__ "(match_len = has_terminal?(#{str}, #{mode}, index))" do
           if address == 0 || decorated? || mode != 'false' || string_length > 1
-	    assign_result "instantiate_node(#{node_class_name},input, index...(index + match_len))"
-	    extend_result_with_inline_module
+            assign_result "instantiate_node(#{node_class_name},input, index...(index + match_len))"
+            # debugger if parent_expression and parent_expression.inline_modules.size > 0
+            # extend_result_with_inline_module parent_expression
+            if parent_expression
+              parent_expression.inline_modules.each do |inline|
+                extend_result inline.module_name
+              end
+            end
           else
             assign_lazily_instantiated_node
-	  end
+          end
           builder << "@index += match_len"
         end
         builder.else_ do
-          builder << "terminal_parse_failure(#{string})"
+          builder << "terminal_parse_failure(#{expected})"
           assign_result 'nil'
         end
       end
+
+      def expected
+        single_quote(string)
+      end
+
+      def inline_module
+        nil
+      end
     end
   end
 end

data/lib/treetop/compiler/ruby_builder.rb

@@ -9,11 +9,11 @@ module Treetop
       def initialize
         @level = 0
         @address_space = LexicalAddressSpace.new
-        @ruby = ""
+        @ruby = String.new("")
       end
       
       def <<(ruby_line)              
-        return if ruby_line.blank?
+        return if ruby_line == ''
         ruby << ruby_line.tabto(level) << "\n"
       end
 

data/lib/treetop/ruby_extensions/string.rb

@@ -13,12 +13,6 @@ class String
     self[0...index].count("\n") + 1
   end
 
-  unless method_defined?(:blank?)
-    def blank?
-      self == ""
-    end
-  end
-
   # The following methods are lifted from Facets 2.0.2
   def tabto(n)
     if self =~ /^( *)\S/