treetop 1.0.2 → 1.1.0

data/Rakefile

@@ -15,13 +15,13 @@ end
 
 gemspec = Gem::Specification.new do |s|
   s.name = "treetop"
-  s.version = "1.0.2"
+  s.version = "1.1.0"
   s.author = "Nathan Sobo"
   s.email = "nathansobo@gmail.com"
   s.homepage = "http://functionalform.blogspot.com"
   s.platform = Gem::Platform::RUBY
   s.summary = "A Ruby-based text parsing and interpretation DSL"
-  s.files = FileList["README", "Rakefile", "{test,lib,bin}/**/*"].to_a
+  s.files = FileList["README", "Rakefile", "{test,lib,bin,examples}/**/*"].to_a
   s.bindir = "bin"
   s.executables = ["tt"]
   s.require_path = "lib"

data/examples/TALK

@@ -0,0 +1,33 @@
+Before talk?
+------------
+- non node-instantiating expressions extend their results with inline modules
+- allow load_grammar and require statements at the top of treetop files
+- deal with facets versioning issue
+
+
+
+Function application... the left-recursion problem
+--------------------------------------------------
+f g x
+
+rule expression
+  application / function / variable
+end
+
+rule application
+  expression space expression
+end
+
+Function application... how can we avoid left recursion?
+--------------------------------------------------------
+rule expression
+  application / function / variable
+end
+
+rule application
+  operator space expression
+end
+
+rule operator
+  function / variable
+end

data/examples/lambda_calculus/arithmetic.treetop

@@ -0,0 +1,99 @@
+grammar Arithmetic  
+  rule expression
+    comparative / additive
+  end
+  
+  rule comparative
+    exp_1:additive space '==' space exp_2:additive {
+      def eval(env={})
+        exp_1.eval(env) == exp_2.eval(env)
+      end
+    }
+  end
+  
+  rule additive
+    operand_1:multitive
+    space operator:additive_op space
+    operand_2:additive <BinaryOperation>
+    /
+    multitive
+  end
+  
+  rule additive_op
+    '+' {
+      def apply(a, b)
+        a + b
+      end
+    }
+    /
+    '-' {
+      def apply(a, b)
+        a - b
+      end
+    }
+  end
+
+  rule multitive
+    operand_1:primary
+    space operator:multitive_op space
+    operand_2:multitive <BinaryOperation>
+    /
+    primary
+  end
+  
+  rule multitive_op
+    '*' {
+      def apply(a, b)
+        a * b
+      end
+    }
+    /
+    '/' {
+      def apply(a, b)
+        a / b
+      end
+    }
+  end  
+  
+  rule primary
+    variable
+    /
+    number
+    /
+    '(' space expression space ')' {
+      def eval(env={})
+        expression.eval(env)
+      end
+    }
+  end
+
+  rule variable
+    [a-z] {
+      def eval(env={})
+        env[name]
+      end
+      
+      def name
+        text_value
+      end
+    }
+  end
+
+  rule number
+    [1-9] [0-9]* {
+      def eval(env={})
+        text_value.to_i
+      end
+    }
+    /
+    '0' {
+      def eval(env={})
+        text_value.to_i
+      end
+    }
+  end
+  
+  rule space
+    ' '*
+  end
+end

data/examples/lambda_calculus/arithmetic_node_classes.rb

@@ -0,0 +1,7 @@
+module Arithmetic
+  class BinaryOperation < Treetop::Runtime::SyntaxNode
+    def eval(env={})
+      operator.apply(operand_1.eval(env), operand_2.eval(env))      
+    end
+  end
+end

data/examples/lambda_calculus/arithmetic_test.rb

@@ -0,0 +1,54 @@
+dir = File.dirname(__FILE__)
+require File.expand_path("#{dir}/test_helper")
+
+require File.expand_path("#{dir}/arithmetic_node_classes")
+load_grammar File.expand_path("#{dir}/arithmetic")
+
+class ArithmeticParserTest < Test::Unit::TestCase
+  include ParserTestHelper
+  
+  def setup
+    @parser = ArithmeticParser.new
+  end
+  
+  def test_number
+    assert_equal 0, parse('0').eval
+    assert_equal 1, parse('1').eval
+    assert_equal 123, parse('123').eval
+  end
+  
+  def test_variable
+    assert_equal 0, parse('x').eval('x' => 0)
+    assert_equal 3, parse('x').eval('x' => 3)
+    assert_equal 10, parse('y').eval('y' => 10)
+  end
+  
+  def test_addition
+    assert_equal 10, parse('x + 5').eval('x' => 5)
+  end
+  
+  def test_subtraction
+    assert_equal 0, parse('x - 5').eval('x' => 5)
+  end
+  
+  def test_multiplication
+    assert_equal 6, parse('x * 2').eval('x' => 3)
+  end
+  
+  def test_division
+    assert_equal 3, parse('x / 2').eval('x' => 6)
+  end
+  
+  def test_order_of_operations
+    assert_equal 11, parse('1 + 2 * 3 + 4').eval
+  end
+  
+  def test_parentheses
+    assert_equal 25, parse('(5 + x) * (10 - y)').eval('x' => 0, 'y' => 5)
+  end
+  
+  def test_equality
+    assert parse('4 == 4').eval
+    assert !parse('4 == 3').eval
+  end
+end

data/examples/lambda_calculus/lambda_calculus.treetop

@@ -0,0 +1,139 @@
+grammar LambdaCalculus
+  include Arithmetic
+
+  rule program
+    expression more_expressions:(';' space expression)* {
+      def eval(env={})
+        env = env.clone
+        last_eval = nil
+        expressions.each do |exp|
+          last_eval = exp.eval(env)
+        end
+        last_eval
+      end
+      
+      def expressions
+        [expression] + more_expressions.elements.map {|elt| elt.expression}
+      end
+    }
+  end
+
+  rule expression
+    definition / conditional / application / function / super
+  end
+  
+  rule definition
+    'def' space variable space expression {
+      def eval(env)
+        env[variable.name] = expression.eval(env)
+      end
+    }
+  end
+  
+  rule conditional
+    'if' space '(' space condition:expression space ')' space
+    true_case:expression space 'else' space false_case:expression {
+      def eval(env)
+        if condition.eval(env)
+          true_case.eval(env)
+        else
+          false_case.eval(env)
+        end
+      end
+    }
+  end
+  
+  rule primary
+    application / super
+  end
+
+  rule application
+    operator space expression <Application> {
+      def eval(env={})
+        left_associative_apply(operator.eval(env), env)
+      end
+      
+      def left_associative_apply(operator, env)
+        if expression.instance_of?(Application)
+          expression.left_associative_apply(operator.apply(expression.operator.eval(env)), env)
+        else
+          operator.apply(expression.eval(env))
+        end
+      end
+      
+      def to_s(env={})
+        operator.to_s(env) + ' ' + expression.to_s(env)
+      end
+    }
+  end
+  
+  rule operator
+    function / variable
+  end
+  
+  rule non_application
+    function / variable
+  end
+  
+  rule function
+    '\\' param:variable '(' body:expression ')' {
+      class Closure
+        attr_reader :env, :function
+        
+        def initialize(function, env)
+          @function = function
+          @env = env
+        end
+      
+        def apply(arg)
+          function.body.eval(function.param.bind(arg, env))
+        end
+      
+        def to_s(other_env={})
+          "\\#{function.param.to_s}(#{function.body.to_s(other_env.merge(env))})"
+        end
+      end
+      
+      def eval(env={})
+        Closure.new(self, env)
+      end
+      
+      def to_s(env={})
+        eval(env).to_s
+      end
+    }
+  end
+  
+  rule variable
+    !keyword [a-z]+ {
+      def eval(env={})
+        env.has_key?(name) ? env[name] : self
+      end
+      
+      def bind(value, env)
+        env.merge(name => value)
+      end
+      
+      def name
+        text_value
+      end
+      
+      def to_s(env={})
+        env.has_key?(name) ? env[name].to_s : name
+      end
+    }
+  end
+  
+  rule keyword
+    ('if' / 'else') !non_space_char
+  end
+  
+  rule non_space_char
+    ![ \n] .
+  end
+  
+  
+  rule space
+    [ \n]*
+  end
+end

data/examples/lambda_calculus/lambda_calculus_node_classes.rb

@@ -0,0 +1,5 @@
+module LambdaCalculus
+  class Application < Treetop::Runtime::SyntaxNode
+    
+  end
+end

data/examples/lambda_calculus/lambda_calculus_test.rb

@@ -0,0 +1,89 @@
+dir = File.dirname(__FILE__)
+require File.expand_path("#{dir}/test_helper")
+require File.expand_path("#{dir}/arithmetic_node_classes")
+require File.expand_path("#{dir}/lambda_calculus_node_classes")
+load_grammar File.expand_path("#{dir}/arithmetic")
+load_grammar File.expand_path("#{dir}/lambda_calculus")
+
+class Treetop::Runtime::SyntaxNode 
+  def method_missing(method, *args)
+    raise "Node representing #{text_value} does not respond to #{method}"
+  end
+end
+
+class LambdaCalculusParserTest < Test::Unit::TestCase
+  include ParserTestHelper
+  
+  def setup
+    @parser = LambdaCalculusParser.new
+  end
+  
+  def test_free_variable
+    assert_equal 'x', parse('x').eval.to_s
+  end
+  
+  def test_variable_binding
+    variable = parse('x').eval
+    env = variable.bind(1, {})
+    assert_equal 1, env['x']
+  end
+  
+  def test_bound_variable_evaluation
+    assert_equal 1, parse('x').eval({'x' => 1})
+  end
+  
+  def test_identity_function
+    assert_equal '\x(x)', parse('\x(x)').eval.to_s
+  end
+  
+  def test_function_returning_constant_function
+    assert_equal '\x(\y(x))', parse('\x(\y(x))').eval.to_s
+  end
+  
+  def test_identity_function_application
+    assert_equal 1, parse('\x(x) 1').eval
+    assert_equal '\y(y)', parse('\x(x) \y(y)').eval.to_s
+  end
+  
+  def test_constant_function_construction
+    assert_equal '\y(1)', parse('\x(\y(x)) 1').eval.to_s
+  end
+  
+  def test_multiple_argument_application_is_left_associative
+    assert_equal '\b(b)', parse('\x(\y(x y)) \a(a) \b(b)').eval.to_s
+  end
+  
+  def test_parentheses_override_application_order
+    assert_equal '\y(\b(b) y)', parse('\x(\y(x y)) (\a(a) \b(b))').eval.to_s
+  end
+  
+  def test_arithmetic_in_function_body
+    assert_equal 10, parse('\x(x + 5) 5').eval
+  end
+  
+  def test_addition_of_function_results
+    assert_equal 20, parse('\x(x + 5) 5 + \x(15 - x) 5').eval
+  end
+  
+  def test_conditional
+    result = parse('if (x) 1 else 2')
+    assert_equal 1, result.eval({'x' => true})
+    assert_equal 2, result.eval({'x' => false})
+  end
+  
+  def test_keyword
+    assert @parser.parse('if').failure?
+    assert @parser.parse('else').failure?
+    assert parse('elsee').success?
+    assert parse('iff').success?
+  end
+  
+  def test_program
+    result = parse('def fact \x(if (x == 0)
+                                  1
+                                else
+                                  x * fact (x - 1));
+                    fact(5)').eval
+    assert_equal 5 * 4 * 3 * 2, result
+  end
+end

data/examples/lambda_calculus/test_helper.rb

@@ -0,0 +1,18 @@
+require 'test/unit'
+require 'rubygems'
+require 'treetop'
+
+module ParserTestHelper
+  def assert_evals_to_self(input)
+    assert_evals_to(input, input)
+  end
+  
+  def parse(input)
+    result = @parser.parse(input)
+    if result.failure?
+      puts result.nested_failures.join("\n")
+    end
+    assert result.success?
+    result
+  end
+end

data/lib/treetop/compiler/metagrammar.rb

@@ -4,7 +4,12 @@ module Treetop
       include ::Treetop::Runtime
       
       def root
-        _nt_treetop_file
+        result = _nt_treetop_file
+        if index == input.size
+          return result
+        else
+          return ParseFailure.new(input, index, result.nested_failures)
+        end
       end
       
       module TreetopFile0
@@ -928,8 +933,8 @@ module Treetop
           atomic
         end
     
-        def node_class
-          node_class_declarations.node_class
+        def node_class_name
+          node_class_declarations.node_class_name
         end
     
         def inline_modules
@@ -956,8 +961,8 @@ module Treetop
           atomic.compile(address, builder, self)
         end
     
-        def node_class
-          node_class_declarations.node_class
+        def node_class_name
+          node_class_declarations.node_class_name
         end
     
         def inline_modules
@@ -1245,8 +1250,8 @@ module Treetop
           suffix.compile(lexical_address, builder, self)
         end
     
-        def node_class
-          'SyntaxNode'
+        def node_class_name
+          nil
         end
     
         def inline_modules
@@ -1381,8 +1386,8 @@ module Treetop
       end
       
       module NodeClassDeclarations1
-        def node_class
-          node_class_expression.node_class
+        def node_class_name
+          node_class_expression.node_class_name
         end
     
         def inline_modules
@@ -2001,14 +2006,14 @@ module Treetop
       end
       
       module NodeClassExpression2
-        def node_class
+        def node_class_name
           elements[2].text_value
         end
       end
       
       module NodeClassExpression3
-        def node_class
-          'SyntaxNode'
+        def node_class_name
+          nil
         end
       end
       

data/lib/treetop/compiler/metagrammar.treetop

@@ -132,8 +132,8 @@ module Treetop
             atomic
           end
       
-          def node_class
-            node_class_declarations.node_class
+          def node_class_name
+            node_class_declarations.node_class_name
           end
       
           def inline_modules
@@ -150,8 +150,8 @@ module Treetop
             atomic.compile(address, builder, self)
           end
       
-          def node_class
-            node_class_declarations.node_class
+          def node_class_name
+            node_class_declarations.node_class_name
           end
       
           def inline_modules
@@ -224,8 +224,8 @@ module Treetop
             suffix.compile(lexical_address, builder, self)
           end
       
-          def node_class
-            'SyntaxNode'
+          def node_class_name
+            nil
           end
       
           def inline_modules
@@ -250,8 +250,8 @@ module Treetop
       
       rule node_class_declarations
         node_class_expression trailing_inline_module {
-          def node_class
-            node_class_expression.node_class
+          def node_class_name
+            node_class_expression.node_class_name
           end
       
           def inline_modules
@@ -318,14 +318,14 @@ module Treetop
 
       rule node_class_expression
         space '<' (!'>' .)+ '>' {
-          def node_class
+          def node_class_name
             elements[2].text_value
           end
         }
         /
         '' {
-          def node_class
-            'SyntaxNode'
+          def node_class_name
+            nil
           end
         }
       end

data/lib/treetop/compiler/node_classes/anything_symbol.rb

@@ -3,7 +3,7 @@ module Treetop
     class AnythingSymbol < AtomicExpression
       def compile(address, builder, parent_expression = nil)
         super
-        assign_result "parse_anything(#{node_class}#{optional_arg(inline_module_name)})"
+        assign_result "parse_anything(#{node_class_name}#{optional_arg(inline_module_name)})"
       end
     end
   end

data/lib/treetop/compiler/node_classes/character_class.rb

@@ -3,7 +3,7 @@ module Treetop
     class CharacterClass < AtomicExpression
       def compile(address, builder, parent_expression = nil)
         super
-        assign_result "parse_char_class(/#{text_value}/, '#{elements[1].text_value.gsub(/'$/, "\\\\'")}', #{node_class}#{optional_arg(inline_module_name)})"
+        assign_result "parse_char_class(/#{text_value}/, '#{elements[1].text_value.gsub(/'$/, "\\\\'")}', #{node_class_name}#{optional_arg(inline_module_name)})"
       end
     end
   end

data/lib/treetop/compiler/node_classes/choice.rb

@@ -15,6 +15,8 @@ module Treetop
         accumulate_nested_result
         builder.if__ subexpression_success? do
           assign_result subexpression_result_var
+          extend_result_with_declared_module
+          extend_result_with_inline_module
           builder << "#{subexpression_result_var}.update_nested_results(#{nested_results_var})"
         end
         builder.else_ do

data/lib/treetop/compiler/node_classes/nonterminal.rb

@@ -5,6 +5,8 @@ module Treetop
         super
         use_vars :result
         assign_result text_value == 'super' ? 'super' : "_nt_#{text_value}"
+        extend_result_with_declared_module
+        extend_result_with_inline_module
       end
     end
   end

data/lib/treetop/compiler/node_classes/parenthesized_expression.rb

@@ -2,7 +2,7 @@ module Treetop
   module Compiler    
     class ParenthesizedExpression < ParsingExpression
       def compile(address, builder, parent_expression = nil)
-        elements[2].compile(address, builder)
+        elements[2].compile(address, builder, parent_expression)
       end
     end
   end

data/lib/treetop/compiler/node_classes/parsing_expression.rb

@@ -9,13 +9,16 @@ module Treetop
         @parent_expression = parent_expression
       end
       
-      def node_class
-        parent_expression && parent_expression.node_class || 'SyntaxNode'
+      def node_class_name
+        parent_expression && parent_expression.node_class_name || 'SyntaxNode'
+      end
+      
+      def declared_module_name
+        parent_expression && parent_expression.node_class_name
       end
       
       def inline_module_name
-        return nil unless parent_expression
-        parent_expression.inline_module_name
+        parent_expression && parent_expression.inline_module_name
       end
       
       def optional_arg(arg)
@@ -70,6 +73,18 @@ module Treetop
       def assign_result(value_ruby)
         builder.assign result_var, value_ruby
       end
+      
+      def extend_result(module_name)
+        builder.extend result_var, module_name
+      end
+
+      def extend_result_with_declared_module
+        extend_result declared_module_name if declared_module_name
+      end
+      
+      def extend_result_with_inline_module
+        extend_result inline_module_name if inline_module_name
+      end
     
       def reset_index
         builder.assign 'self.index', start_index_var

data/lib/treetop/compiler/node_classes/repetition.rb

@@ -25,8 +25,8 @@ module Treetop
       end
       
       def assign_and_extend_result
-        assign_result "#{node_class}.new(input, #{start_index_var}...index, #{accumulator_var}, #{nested_results_var})"
-        builder << "#{result_var}.extend(#{inline_module_name})" if inline_module_name
+        assign_result "#{node_class_name}.new(input, #{start_index_var}...index, #{accumulator_var}, #{nested_results_var})"
+        extend_result_with_inline_module
       end
     end
 

data/lib/treetop/compiler/node_classes/sequence.rb

@@ -7,9 +7,9 @@ module Treetop
         use_vars :result, :start_index, :accumulator, :nested_results
         compile_sequence_elements(sequence_elements)
         builder.if__ "#{accumulator_var}.last.success?" do
-          assign_result "(#{node_class_declarations.node_class}).new(input, #{start_index_var}...index, #{accumulator_var})"
-          builder << "#{result_var}.extend(#{sequence_element_accessor_module_name})" if sequence_element_accessor_module_name
-          builder << "#{result_var}.extend(#{inline_module_name})" if inline_module_name
+          assign_result "(#{node_class_name}).new(input, #{start_index_var}...index, #{accumulator_var})"
+          extend_result sequence_element_accessor_module_name if sequence_element_accessor_module_name
+          extend_result_with_inline_module
         end
         builder.else_ do
           reset_index
@@ -18,6 +18,10 @@ module Treetop
         end_comment(self)
       end
       
+      def node_class_name
+        node_class_declarations.node_class_name || 'SyntaxNode'
+      end
+      
       def compile_sequence_elements(elements)
         obtain_new_subexpression_address
         elements.first.compile(subexpression_address, builder)

data/lib/treetop/compiler/node_classes/terminal.rb

@@ -3,7 +3,7 @@ module Treetop
     class Terminal < AtomicExpression
       def compile(address, builder, parent_expression = nil)
         super
-        assign_result "parse_terminal(#{text_value}, #{node_class}#{optional_arg(inline_module_name)})"
+        assign_result "parse_terminal(#{text_value}, #{node_class_name}#{optional_arg(inline_module_name)})"
       end
     end
  end

data/lib/treetop/compiler/ruby_builder.rb

@@ -51,6 +51,10 @@ module Treetop
         end
       end
       
+      def extend(var, module_name)
+        self << "#{var}.extend(#{module_name})"
+      end
+      
       def accumulate(left, right)
         self << "#{left} << #{right}"
       end

data/test/compiler/choice_test.rb

@@ -46,4 +46,29 @@ describe "A choice between sequences", :extend => CompilerTestCase do
     parse('foobarbaz').should be_success
     parse('bingbangboom').should be_success
   end
-end
+end
+
+describe "A choice between terminals followed by a block", :extend => CompilerTestCase do  
+  testing_expression "('a'/ 'b' / 'c') { def a_method; end }"
+
+  it "extends a match of any of its subexpressions with a module created from the block" do
+    ['a', 'b', 'c'].each do |letter|
+      parse(letter).should respond_to(:a_method)
+    end
+  end
+end
+
+class ChoiceWithDeclaredModuleTest < CompilerTestCase
+  module TestModule
+    def a_method
+    end
+  end
+  
+  testing_expression "('a'/ 'b' / 'c') <TestModule>"
+
+  it "extends a match of any of its subexpressions with a module created from the block" do
+    ['a', 'b', 'c'].each do |letter|
+      parse(letter).should respond_to(:a_method)
+    end
+  end
+end

data/test/compiler/nonterminal_symbol_test.rb

@@ -1,7 +1,7 @@
 require File.join(File.dirname(__FILE__), '..', 'test_helper')
 
-describe "A nonterminal symbol", :extend => CompilerTestCase do
-  testing_expression 'foo'
+describe "A nonterminal symbol followed by a block", :extend => CompilerTestCase do
+  testing_expression 'foo { def a_method; end }'
   
   parser_class_under_test.class_eval do
     def _nt_foo
@@ -9,7 +9,30 @@ describe "A nonterminal symbol", :extend => CompilerTestCase do
     end
   end
   
-  it "compiles to a message send" do
-    parse('').should == '_nt_foo called'
+  it "compiles to a method call, extending its results with the anonymous module for the block" do
+    result = parse('')
+    result.should == '_nt_foo called'
+    result.should respond_to(:a_method)
   end
-end
+end
+
+class NonterminalWithModuleDeclationTest < CompilerTestCase
+  module TestModule
+    def a_method
+    end
+  end
+  
+  testing_expression 'foo <TestModule>'
+  
+  parser_class_under_test.class_eval do
+    def _nt_foo
+      '_nt_foo called'
+    end
+  end
+  
+  it "compiles to a method call, extending its results with the anonymous module for the block" do
+    result = parse('')
+    result.should == '_nt_foo called'
+    result.should respond_to(:a_method)
+  end
+end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: treetop
 version: !ruby/object:Gem::Version 
-  version: 1.0.2
-date: 2007-10-22 00:00:00 -07:00
+  version: 1.1.0
+date: 2007-10-25 00:00:00 -07:00
 summary: A Ruby-based text parsing and interpretation DSL
 require_paths: 
 - lib
@@ -144,6 +144,15 @@ files:
 - lib/treetop/runtime.rb
 - lib/treetop.rb
 - bin/tt
+- examples/lambda_calculus
+- examples/lambda_calculus/arithmetic.treetop
+- examples/lambda_calculus/arithmetic_node_classes.rb
+- examples/lambda_calculus/arithmetic_test.rb
+- examples/lambda_calculus/lambda_calculus.treetop
+- examples/lambda_calculus/lambda_calculus_node_classes.rb
+- examples/lambda_calculus/lambda_calculus_test.rb
+- examples/lambda_calculus/test_helper.rb
+- examples/TALK
 test_files: []
 
 rdoc_options: []