treetop 1.3.0 → 1.4.1

data/lib/treetop/compiler/metagrammar.treetop

@@ -127,6 +127,18 @@ module Treetop
           end
         }
         /
+	prefix space? predicate_block {
+          def compile(address, builder, parent_expression=nil)
+            prefix.compile(address, builder, self)
+          end
+          def prefixed_expression
+            predicate_block
+          end
+          def inline_modules
+            []
+          end
+        }
+        /
         atomic suffix node_class_declarations {
           def compile(address, builder, parent_expression=nil)
             suffix.compile(address, builder, self)
@@ -223,6 +235,18 @@ module Treetop
           end
         }
         /
+        prefix space? predicate_block {
+          def compile(address, builder, parent_expression=nil)
+            prefix.compile(address, builder, self)
+          end
+          def prefixed_expression
+            predicate_block
+          end
+          def inline_modules
+            []
+          end
+        }
+        /
         atomic suffix {
           def compile(lexical_address, builder)
             suffix.compile(lexical_address, builder, self)
@@ -321,7 +345,7 @@ module Treetop
       end
 
       rule character_class
-	'[' characters:(!']' ('\\' . /!'\\' .))+ ']' <CharacterClass> {
+        '[' characters:(!']' ('\\' . /!'\\' .))+ ']' <CharacterClass> {
           def characters
             super.text_value
           end
@@ -372,6 +396,10 @@ module Treetop
         }
       end
 
+      rule predicate_block
+        '' inline_module <PredicateBlock>
+      end
+
       rule inline_module
         '{' (inline_module / ![{}] .)* '}' <InlineModule>
       end

data/lib/treetop/compiler/node_classes/anything_symbol.rb

@@ -4,9 +4,10 @@ module Treetop
       def compile(address, builder, parent_expression = nil)
         super
         builder.if__ "index < input_length" do
-          assign_result "instantiate_node(#{node_class_name},input, index...(index + 1))"
+          builder << 'next_character = index + input[index..-1].match(/\A(.)/um).end(1)'
+          assign_result "instantiate_node(#{node_class_name},input, index...next_character)"
           extend_result_with_inline_module
-          builder << "@index += 1"
+          builder << "@index = next_character"
         end
         builder.else_ do
           builder << 'terminal_parse_failure("any character")'

data/lib/treetop/compiler/node_classes/character_class.rb

@@ -5,12 +5,17 @@ module Treetop
         super
         
         builder.if__ "has_terminal?(#{grounded_regexp(text_value)}, true, index)" do
-          assign_result "instantiate_node(#{node_class_name},input, index...(index + 1))"
-          extend_result_with_inline_module
-          builder << "@index += 1"
+          builder << 'next_character = index + input[index..-1].match(/\A(.)/um).end(1)'
+          if address == 0 || decorated?
+            assign_result "instantiate_node(#{node_class_name}, input, index...next_character)"
+            extend_result_with_inline_module
+          else
+            assign_lazily_instantiated_node
+          end
+          builder << "@index = next_character"
         end
         builder.else_ do
-          "terminal_parse_failure(#{single_quote(characters)})"
+          # "terminal_parse_failure(#{single_quote(characters)})"
           assign_result 'nil'
         end
       end

data/lib/treetop/compiler/node_classes/parsing_expression.rb

@@ -21,6 +21,10 @@ module Treetop
         parent_expression && parent_expression.inline_module_name
       end
       
+      def decorated?
+        parent_expression && (parent_expression.node_class_name || parent_expression.node_class_name || parent_expression.inline_module_name)
+      end
+      
       def optional_arg(arg)
         if arg
           ", #{arg}"
@@ -89,6 +93,10 @@ module Treetop
       def assign_failure(start_index_var)
         assign_result("nil")
       end
+      
+      def assign_lazily_instantiated_node
+        assign_result("true")
+      end
     
       def var_initialization
         left, right = [], []

data/lib/treetop/compiler/node_classes/predicate_block.rb

@@ -0,0 +1,16 @@
+module Treetop
+  module Compiler
+    class PredicateBlock < ParsingExpression
+      def compile(index, builder, parent_expression = nil)
+        super
+        # REVISIT: This is distinctly dodgey, but since we can only be called from
+        # two contexts, and it works in both those, I'm going with it for now, as
+        # opposed to doing the major refactor of providing a proper way of accessing
+        # the parent's accumulator variable.
+        p = parent
+        p = p.parent while p && !p.respond_to?(:accumulator_var)
+        assign_result "lambda #{text_value}.call(#{p ? p.accumulator_var : ""})"
+      end
+    end
+  end
+end

data/lib/treetop/compiler/node_classes/sequence.rb

@@ -53,9 +53,12 @@ module Treetop
       def compile(index, builder, rule)
         super
         builder.module_declaration(module_name) do
+          elements_by_name = sequence_elements.inject({}){|h,e| (h[e.label_name] ||= []) << e; h}
           sequence_elements.each_with_index do |element, index|
             if element.label_name
-              builder.method_declaration(element.label_name) do
+              repetitions = elements_by_name[element.label_name]
+              label_name = element.label_name + (repetitions.size > 1 ? (repetitions.index(element)+1).to_s : "")
+              builder.method_declaration(label_name) do
                 builder << "elements[#{index}]"
               end
               builder.newline unless index == sequence_elements.size - 1

data/lib/treetop/compiler/node_classes.rb

@@ -2,6 +2,7 @@ dir = File.dirname(__FILE__)
 require File.join(dir, *%w[node_classes parsing_expression])
 require File.join(dir, *%w[node_classes atomic_expression])
 require File.join(dir, *%w[node_classes inline_module])
+require File.join(dir, *%w[node_classes predicate_block])
 require File.join(dir, *%w[node_classes treetop_file])
 require File.join(dir, *%w[node_classes grammar])
 require File.join(dir, *%w[node_classes declaration_sequence])

data/lib/treetop/compiler.rb

@@ -2,5 +2,5 @@ dir = File.dirname(__FILE__)
 require File.join(dir, *%w[compiler lexical_address_space])
 require File.join(dir, *%w[compiler ruby_builder])
 require File.join(dir, *%w[compiler node_classes])
-require File.join(dir, *%w[compiler metagrammar]) unless $exclude_metagrammar
+require File.join(dir, *%w[compiler metagrammar]) unless defined?($exclude_metagrammar)
 require File.join(dir, *%w[compiler grammar_compiler])

data/lib/treetop/ruby_extensions/array.rb

@@ -0,0 +1,22 @@
+class Array
+	def join_with(method, pattern = "")
+		return join(pattern) unless method
+		return "" if self.length == 0
+
+		args = []
+		if method.respond_to? :to_hash
+			args = method[:args] || []
+			method = method[:name]
+		end
+
+		output = self[0].send(method, *args)
+		for i in (1...self.length)
+			output += pattern + self[i].send(method, *args)
+		end
+		output
+	end
+
+	def to_tt
+		self.join_with({:name => :seq_to_tt, :args => [true]}, " ")
+	end
+end

data/lib/treetop/ruby_extensions/nil.rb

@@ -0,0 +1,5 @@
+class NilClass
+	def to_tt
+		""
+	end
+end

data/lib/treetop/ruby_extensions/object.rb

@@ -0,0 +1,57 @@
+class Object
+	def sequence
+		@sequence ||= []
+	end
+
+	def /(other)
+		sequence.push(other)
+		self
+	end
+
+	def seq_to_tt(inline = false)
+		separator = inline ? " / " : "\n/\n"
+		tt = if sequence.length == 0
+			self.to_tt
+		else
+			output = self.to_tt + separator + sequence.join_with({:name => :seq_to_tt, :args => [true]}, separator)
+			output = "( #{output} )" if inline
+			output
+		end
+
+		# Operators
+		tt = "&" + tt if @amper
+		tt = "!" + tt if @bang
+		tt += "*" if @kleene
+		tt += "+" if @plus
+		tt += "?" if @mark
+
+		tt += " <#{@node.to_s}>" if @node
+		tt += " {\n#{@block.gsub("\t", "  ").justify.indent_paragraph(2)}\n}" if @block
+		tt = @label.to_s + ':' + tt if @label
+		tt
+	end
+
+	def node(name)
+		@node = name
+		self
+	end
+
+	def block(content)
+		@block = content
+		self
+	end
+
+	def label(name)
+		@label = name
+		self
+	end
+
+	[:mark, :kleene, :plus, :amper, :bang].each do |sym|
+		Object.class_eval(%{
+			def #{sym.to_s}
+				@#{sym.to_s} = true
+				self
+			end
+		})
+	end
+end

data/lib/treetop/ruby_extensions/regexp.rb

@@ -0,0 +1,5 @@
+class Regexp
+	def to_tt
+		self.inspect
+	end
+end

data/lib/treetop/ruby_extensions/string.rb

@@ -8,7 +8,7 @@ class String
       index + 1
     end
   end
-  
+
   def line_of(index)
     self[0...index].count("\n") + 1
   end
@@ -36,7 +36,33 @@ class String
     end
   end
 
+	def indent_paragraph(n)
+		out = ""
+		self.each_line {|line| out += line.indent(n) }
+		out
+	end
+
+	# Removes indentation uniformly.
+	def justify
+		min = self.length
+		self.each_line {|line|
+			next if line.strip == ""
+			if line =~ /^(  *)\S/
+				min = $1.length if min > $1.length
+			else
+				min = 0
+			end
+		}
+		out = ""
+		self.each_line {|line| out += line.slice(min...line.length) || "\n" }
+		out.strip
+	end
+
   def treetop_camelize
     to_s.gsub(/\/(.?)/){ "::" + $1.upcase }.gsub(/(^|_)(.)/){ $2.upcase }
   end
-end
+
+	def to_tt
+		"'#{self}'"
+	end
+end

data/lib/treetop/ruby_extensions/symbol.rb

@@ -0,0 +1,5 @@
+class Symbol
+	def to_tt
+		self.to_s
+	end
+end

data/lib/treetop/ruby_extensions.rb

@@ -1,2 +1,4 @@
 dir = File.dirname(__FILE__)
-require "#{dir}/ruby_extensions/string"
+Dir.glob("#{dir}/ruby_extensions/*.rb").each do |file|
+	require file
+end

data/lib/treetop/runtime/compiled_parser.rb

@@ -3,7 +3,7 @@ module Treetop
     class CompiledParser
       include Treetop::Runtime
       
-      attr_reader :input, :index, :terminal_failures, :max_terminal_failure_index
+      attr_reader :input, :index, :max_terminal_failure_index
       attr_writer :root
       attr_accessor :consume_all_input
       alias :consume_all_input? :consume_all_input
@@ -25,22 +25,26 @@ module Treetop
       end
 
       def failure_line
-        terminal_failures && input.line_of(failure_index)
+        @terminal_failures && input.line_of(failure_index)
       end
 
       def failure_column
-        terminal_failures && input.column_of(failure_index)
+        @terminal_failures && input.column_of(failure_index)
       end
 
       def failure_reason
         return nil unless (tf = terminal_failures) && tf.size > 0
-	"Expected " +
-	  (tf.size == 1 ?
-	   tf[0].expected_string :
-           "one of #{tf.map{|f| f.expected_string}.uniq*', '}"
-	  ) +
-          " at line #{failure_line}, column #{failure_column} (byte #{failure_index+1})" +
-          " after #{input[index...failure_index]}"
+        "Expected " +
+          (tf.size == 1 ?
+           tf[0].expected_string :
+                 "one of #{tf.map{|f| f.expected_string}.uniq*', '}"
+          ) +
+                " at line #{failure_line}, column #{failure_column} (byte #{failure_index+1})" +
+                " after #{input[index...failure_index]}"
+      end
+      
+      def terminal_failures
+        @terminal_failures.map! {|tf_ary| TerminalParseFailure.new(*tf_ary) }
       end
 
 
@@ -84,7 +88,7 @@ module Treetop
     
       def has_terminal?(terminal, regex, index)
         if regex
-          rx = @regexps[terminal] ||= Regexp.new(terminal)
+          rx = @regexps[terminal] ||= Regexp.new(terminal, nil, 'u')
           input.index(rx, index) == index
         else
           input[index, terminal.size] == terminal
@@ -97,7 +101,7 @@ module Treetop
           @max_terminal_failure_index = index
           @terminal_failures = []
         end
-        terminal_failures << TerminalParseFailure.new(index, expected_string)
+        @terminal_failures << [index, expected_string]
         return nil
       end
     end

data/lib/treetop/runtime/syntax_node.rb

@@ -1,17 +1,35 @@
 module Treetop
   module Runtime
     class SyntaxNode
-      attr_reader :input, :interval, :elements
+      attr_reader :input, :interval
       attr_accessor :parent
+      attr_reader :dot_id
+
+      @@dot_id_counter = 0
 
       def initialize(input, interval, elements = nil)
         @input = input
         @interval = interval
-        if @elements = elements
-          elements.each do |element|
-            element.parent = self
+        @elements = elements
+      end
+
+      def elements
+        return @elements if terminal?
+        # replace the character class placeholders in the sequence (lazy instantiation)
+        last_element = nil
+        @comprehensive_elements ||= @elements.map do |element|
+          if element == true
+            index = last_element ? last_element.interval.last : interval.first
+            element = SyntaxNode.new(input, index...(index + 1))
           end
+          element.parent = self
+          last_element = element
         end
+
+        @dot_id = @@dot_id_counter
+        @@dot_id_counter += 1
+
+        @comprehensive_elements
       end
 
       def terminal?
@@ -29,6 +47,10 @@ module Treetop
       def empty?
         interval.first == interval.last && interval.exclude_end?
       end
+      
+      def <=>(other)
+        self.interval.first <=> other.interval.first
+      end
 
       def extension_modules
         local_extensions =
@@ -57,7 +79,7 @@ module Treetop
           im +
           (elements && elements.size > 0 ?
             ":" +
-              (@elements||[]).map{|e|
+              (elements||[]).map{|e|
           begin
             "\n"+e.inspect(indent+"  ")
           rescue  # Defend against inspect not taking a parameter
@@ -67,6 +89,26 @@ module Treetop
             ""
           )
       end
+
+      def write_dot(io)
+        io.puts "node#{dot_id} [label=\"#{text_value}\"];"
+        if nonterminal? then
+          elements.each do
+            |x|
+            io.puts "node#{dot_id} -> node#{x.dot_id};"
+            x.write_dot(io)
+          end
+        end
+      end
+
+      def write_dot_file(fname)
+        File.open(fname + ".dot","w") do
+          |file|
+          file.puts "digraph G {"
+          write_dot(file)
+          file.puts "}"
+        end
+      end
     end
   end
 end

data/lib/treetop/syntax.rb

@@ -0,0 +1,39 @@
+=begin rdoc
+	Definition of TreeTop syntax in pure Ruby.
+=end
+
+module Treetop
+	# Provides the possibility to write Treetop syntax as a Ruby code.
+	# Symbols act as nonterminals, strings as terminals, arrays as
+	# sequences. Ordered choices are defined similar to original Treetop
+	# syntax.
+	#
+	# (Note: it is better not to use numbers; use Strings instead)
+	#
+	module Syntax
+		class Grammar
+			attr_reader :source
+			def initialize
+				@source = ""
+			end
+
+			def rule(name)
+				@source += "rule #{name.to_s}\n#{yield.seq_to_tt.indent_paragraph(2)}\nend\n"
+			end
+
+			def include(name)
+				@source += "include #{name.to_s}\n"
+			end
+		end
+
+		def grammar(name, &block)
+			Syntax.grammar(name, &block)
+		end
+
+		def self.grammar(name, &block)
+			(g = Grammar.new).instance_eval(&block)
+			source = "grammar #{name.to_s}\n#{g.source.indent_paragraph(2)}end\n"
+			Treetop.load_from_string(source)
+		end
+	end
+end

data/lib/treetop/version.rb

@@ -1,8 +1,8 @@
 module Treetop #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
-    MINOR = 3
-    TINY  = 0
+    MINOR = 4
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/treetop.rb

@@ -1,11 +1,17 @@
 require 'rubygems'
 
+module Treetop
+  VALID_GRAMMAR_EXT = ['treetop', 'tt']
+  VALID_GRAMMAR_EXT_REGEXP = /\.(#{VALID_GRAMMAR_EXT.join('|')})\Z/o
+end
+
 dir = File.dirname(__FILE__)
 
 TREETOP_ROOT = File.join(dir, 'treetop')
 require File.join(TREETOP_ROOT, "ruby_extensions")
 require File.join(TREETOP_ROOT, "runtime")
 require File.join(TREETOP_ROOT, "compiler")
+require File.join(TREETOP_ROOT, "syntax")
 
 require 'polyglot'
-Polyglot.register(["treetop", "tt"], Treetop)
+Polyglot.register(Treetop::VALID_GRAMMAR_EXT, Treetop)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: treetop
 version: !ruby/object:Gem::Version 
-  version: 1.3.0
+  version: 1.4.1
 platform: ruby
 authors: 
 - Nathan Sobo
@@ -9,7 +9,7 @@ autorequire: treetop
 bindir: bin
 cert_chain: []
 
-date: 2009-07-22 00:00:00 +10:00
+date: 2009-09-04 00:00:00 +10:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -32,7 +32,7 @@ extra_rdoc_files: []
 
 files: 
 - LICENSE
-- README
+- README.md
 - Rakefile
 - lib/treetop/bootstrap_gen_1_metagrammar.rb
 - lib/treetop/compiler/grammar_compiler.rb
@@ -52,6 +52,7 @@ files:
 - lib/treetop/compiler/node_classes/parsing_expression.rb
 - lib/treetop/compiler/node_classes/parsing_rule.rb
 - lib/treetop/compiler/node_classes/predicate.rb
+- lib/treetop/compiler/node_classes/predicate_block.rb
 - lib/treetop/compiler/node_classes/repetition.rb
 - lib/treetop/compiler/node_classes/sequence.rb
 - lib/treetop/compiler/node_classes/terminal.rb
@@ -60,7 +61,12 @@ files:
 - lib/treetop/compiler/node_classes.rb
 - lib/treetop/compiler/ruby_builder.rb
 - lib/treetop/compiler.rb
+- lib/treetop/ruby_extensions/array.rb
+- lib/treetop/ruby_extensions/nil.rb
+- lib/treetop/ruby_extensions/object.rb
+- lib/treetop/ruby_extensions/regexp.rb
 - lib/treetop/ruby_extensions/string.rb
+- lib/treetop/ruby_extensions/symbol.rb
 - lib/treetop/ruby_extensions.rb
 - lib/treetop/runtime/compiled_parser.rb
 - lib/treetop/runtime/interval_skip_list/head_node.rb
@@ -72,6 +78,7 @@ files:
 - lib/treetop/runtime/terminal_parse_failure_debug.rb
 - lib/treetop/runtime/terminal_syntax_node.rb
 - lib/treetop/runtime.rb
+- lib/treetop/syntax.rb
 - lib/treetop/version.rb
 - lib/treetop.rb
 - bin/tt
@@ -94,6 +101,8 @@ files:
 - examples/lambda_calculus/lambda_calculus_node_classes.rb
 - examples/lambda_calculus/lambda_calculus_test.rb
 - examples/lambda_calculus/test_helper.rb
+- examples/ruby_syntax/syntax_test.rb
+- examples/ruby_syntax/test_helper.rb
 has_rdoc: true
 homepage: http://functionalform.blogspot.com
 licenses: []
@@ -118,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.4
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: A Ruby-based text parsing and interpretation DSL