dhaka 1.0.0 → 2.0.0

data/test/all_tests.rb

@@ -1,18 +1,19 @@
 #!/usr/bin/env ruby
 require 'test/unit'
-
 require 'grammar_test'
+require 'malformed_grammar_test'
 require 'parser_test'
-require 'arithmetic_evaluator_test'
-require 'compiled_parser_test'
 require 'evaluator_test'
+require 'compiled_parser_test'
+require 'arithmetic_evaluator_test'
+require 'arithmetic_grammar_test'
 require 'arithmetic_tokenizer_test'
-require 'malformed_grammar_test'
-require 'brackets_test'
 require 'arithmetic_precedence_grammar_test'
 require 'arithmetic_precedence_parser_test'
+require 'brackets_test'
 require 'precedence_grammar_test'
 require 'chittagong_parser_test'
 require 'chittagong_evaluator_test'
 require 'chittagong_tokenizer_test'
-require 'chittagong_test'
+require 'chittagong_test'
+require 'chittagong_driver_test'

data/test/arithmetic_evaluator_test.rb

@@ -11,43 +11,43 @@ class TestArithmeticEvaluator < Test::Unit::TestCase
   end
 
 
-  def test_results_simple_arithmetic_given_tokens_and_syntax_tree_1
+  def test_results_simple_arithmetic_given_tokens_and_parse_tree_1
     
-    token_stream = [token('n', 2), token('-', nil), token('n', 4)]
-    syntax_tree = parse(token_stream)
-    assert_equal -2, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
+    token_stream = [token('n', 2), token('-', nil), token('n', 4), token(Dhaka::END_SYMBOL_NAME, nil)]
+    parse_tree = parse(token_stream)
+    assert_equal -2, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(parse_tree)
     
   end
 
-  def test_results_simple_arithmetic_given_tokens_and_syntax_tree_2
+  def test_results_simple_arithmetic_given_tokens_and_parse_tree_2
 
-    token_stream = [token('n', 2), token('-', nil), token('(', nil), token('n', 3), token('/', nil), token('n', 4), token(')', nil)]
-    syntax_tree = parse(token_stream)
-    assert_equal 1.25, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
+    token_stream = [token('n', 2), token('-', nil), token('(', nil), token('n', 3), token('/', nil), token('n', 4), token(')', nil), token(Dhaka::END_SYMBOL_NAME, nil)]
+    parse_tree = parse(token_stream)
+    assert_equal 1.25, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(parse_tree)
 
   end
 
-  def test_results_simple_arithmetic_given_tokens_and_syntax_tree_3
+  def test_results_simple_arithmetic_given_tokens_and_parse_tree_3
 
-    token_stream = [token('n', 2), token('+', nil), token('(', nil), token('n', 3), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil)]
-    syntax_tree = parse(token_stream)
-    assert_equal 3.5, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
+    token_stream = [token('n', 2), token('+', nil), token('(', nil), token('n', 3), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil), token(Dhaka::END_SYMBOL_NAME, nil)]
+    parse_tree = parse(token_stream)
+    assert_equal 3.5, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(parse_tree)
 
   end
 
-  def test_results_simple_arithmetic_given_tokens_and_syntax_tree_4
+  def test_results_simple_arithmetic_given_tokens_and_parse_tree_4
 
-    token_stream = [token('n', 2), token('+', nil), token('h', nil), token('(', nil), token('n', 3), token(',', nil), token('n', 4), token(')', nil)]
-    syntax_tree = parse(token_stream)
-    assert_equal 6, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
+    token_stream = [token('n', 2), token('+', nil), token('h', nil), token('(', nil), token('n', 3), token(',', nil), token('n', 4), token(')', nil), token(Dhaka::END_SYMBOL_NAME, nil)]
+    parse_tree = parse(token_stream)
+    assert_equal 6, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(parse_tree)
 
   end
 
-  def test_results_simple_arithmetic_given_tokens_and_syntax_tree_5
+  def test_results_simple_arithmetic_given_tokens_and_parse_tree_5
 
-    token_stream = [token('n', 2), token('+', nil), token('l', nil), token('(', nil), token('n', 3), token(',', nil), token('n', 4), token(')', nil)]
-    syntax_tree = parse(token_stream)
-    assert_equal 5, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
+    token_stream = [token('n', 2), token('+', nil), token('l', nil), token('(', nil), token('n', 3), token(',', nil), token('n', 4), token(')', nil), token(Dhaka::END_SYMBOL_NAME, nil)]
+    parse_tree = parse(token_stream)
+    assert_equal 5, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(parse_tree)
 
   end
 

data/test/arithmetic_precedence_evaluator.rb

@@ -24,7 +24,7 @@ class ArithmeticPrecedenceEvaluator < Dhaka::Evaluator
     end
 
     for_literal do
-      child_nodes[0].token.value
+      child_nodes[0].token.value.to_i
     end
 
     for_parenthetized_expression do

data/test/arithmetic_precedence_parser_test.rb

@@ -13,11 +13,11 @@ class TestArithmeticPrecedenceParser < Test::Unit::TestCase
     assert_equal(30, fake_logger.warnings.size)
     assert_equal(0, fake_logger.errors.size)
     
-    assert_equal(-8, evaluate(parse("5 * -14/(2*7 - 7) + 2").syntax_tree))
-    assert_equal(-4, evaluate(parse("-2^2").syntax_tree))
-    assert_equal(10, evaluate(parse("2+2^3").syntax_tree))
-    assert_equal(64, evaluate(parse("(2+2)^3").syntax_tree))
-    assert_equal(128, evaluate(parse("(2+2)^3*2").syntax_tree))
+    assert_equal(-8, evaluate(parse("5 * -14/(2*7 - 7) + 2").parse_tree))
+    assert_equal(-4, evaluate(parse("-2^2").parse_tree))
+    assert_equal(10, evaluate(parse("2+2^3").parse_tree))
+    assert_equal(64, evaluate(parse("(2+2)^3").parse_tree))
+    assert_equal(128, evaluate(parse("(2+2)^3*2").parse_tree))
     assert(parse("(2+2)^3^2").has_error?)
   end
   
@@ -25,8 +25,8 @@ class TestArithmeticPrecedenceParser < Test::Unit::TestCase
     ArithmeticPrecedenceParser.parse(ArithmeticPrecedenceTokenizer.tokenize(input))
   end
 
-  def evaluate(syntax_tree)
-    ArithmeticPrecedenceEvaluator.new.evaluate(syntax_tree)
+  def evaluate(parse_tree)
+    ArithmeticPrecedenceEvaluator.new.evaluate(parse_tree)
   end
 
 end

data/test/arithmetic_precedence_tokenizer.rb

@@ -15,11 +15,11 @@ class ArithmeticPrecedenceTokenizer < Dhaka::Tokenizer
 
   for_state Dhaka::TOKENIZER_IDLE_STATE do
     for_characters(all_characters - (digits + whitespace)) do
-      tokens << Dhaka::Token.new(ArithmeticPrecedenceGrammar.symbol_for_name(curr_char), nil)
+      create_token(curr_char, nil)
       advance
     end
     for_characters digits do
-      self.accumulator = ''
+      create_token('n', '')
       switch_to :get_integer_literal
     end
     for_character whitespace do
@@ -29,19 +29,13 @@ class ArithmeticPrecedenceTokenizer < Dhaka::Tokenizer
   
   for_state :get_integer_literal do
     for_characters all_characters - digits do
-      tokens << integer_literal_token(accumulator.to_i)
       switch_to Dhaka::TOKENIZER_IDLE_STATE
     end
     for_characters digits do
-      self.accumulator += curr_char
+      curr_token.value += curr_char
       advance
-      tokens << integer_literal_token(accumulator.to_i) unless curr_char
     end
   end
   
-  def integer_literal_token(value)
-    Dhaka::Token.new(ArithmeticPrecedenceGrammar.symbol_for_name('n'), value)
-  end
-  
 end
 

data/test/arithmetic_test_methods.rb

@@ -1,11 +1,11 @@
 module ArithmeticTestMethods
   
   def parse(token_stream)
-    CompiledArithmeticParser.parse(token_stream).syntax_tree
+    CompiledArithmeticParser.parse(token_stream).parse_tree
   end
   
   def token(symbol_name, value)
-    Dhaka::Token.new(CompiledArithmeticParser.grammar.symbol_for_name(symbol_name), value)
+    Dhaka::Token.new(symbol_name, value, nil)
   end
   
 end

data/test/arithmetic_tokenizer.rb

@@ -15,11 +15,11 @@ class ArithmeticTokenizer < Dhaka::Tokenizer
 
   for_state Dhaka::TOKENIZER_IDLE_STATE do
     for_characters(all_characters - (digits + whitespace)) do
-      tokens << Dhaka::Token.new(ArithmeticGrammar.symbol_for_name(curr_char), nil)
+      create_token(curr_char, nil)
       advance
     end
     for_characters digits do
-      self.accumulator = ''
+      create_token('n', '')
       switch_to :get_integer_literal
     end
     for_character whitespace do
@@ -29,19 +29,13 @@ class ArithmeticTokenizer < Dhaka::Tokenizer
   
   for_state :get_integer_literal do
     for_characters all_characters - digits do
-      tokens << integer_literal_token(accumulator.to_i)
       switch_to Dhaka::TOKENIZER_IDLE_STATE
     end
     for_characters digits do
-      self.accumulator += curr_char
+      curr_token.value += curr_char
       advance
-      tokens << integer_literal_token(accumulator.to_i) unless curr_char
     end
   end
   
-  def integer_literal_token(value)
-    Dhaka::Token.new(ArithmeticGrammar.symbol_for_name('n'), value)
-  end
-  
 end
 

data/test/arithmetic_tokenizer_test.rb

@@ -2,31 +2,35 @@ require "test/unit"
 require "arithmetic_tokenizer"
 
 class TestArithmeticTokenizer < Test::Unit::TestCase
-  def test_returns_empty_array_for_empty_input
-    assert_equal([], ArithmeticTokenizer.tokenize([]))
+  def test_returns_end_of_input_token_for_empty_input
+    assert_equal([token(Dhaka::END_SYMBOL_NAME, nil)], ArithmeticTokenizer.tokenize([]).to_a)
   end
   def test_tokenizes_given_a_string_input
-    assert_equal([token('n', 2), token('-', nil), token('n', 4)], ArithmeticTokenizer.tokenize('2 - 4')) 
+    assert_equal([token('n', 2), token('-', nil), token('n', 4), token(Dhaka::END_SYMBOL_NAME, nil)], ArithmeticTokenizer.tokenize('2 - 4').to_a) 
   end
   def test_a_longer_input
-    actual = ArithmeticTokenizer.tokenize('2+(3 / (7 - 5))')
-    assert_equal([token('n', 2), token('+', nil), token('(', nil), token('n', 3), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil)], actual)
+    actual = ArithmeticTokenizer.tokenize('2+(3 / (7 - 5))').to_a
+    assert_equal([token('n', 2), token('+', nil), token('(', nil), token('n', 3), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil), token(Dhaka::END_SYMBOL_NAME, nil)], actual)
   end
 
   def test_another_input_with_multi_digit_numbers
-    actual = ArithmeticTokenizer.tokenize('2034 +(3433 / (7 - 5))')
-    assert_equal([token('n', 2034), token('+', nil), token('(', nil), token('n', 3433), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil)], actual)
+    actual = ArithmeticTokenizer.tokenize('2034 +(3433 / (7 - 5))').to_a
+    assert_equal([token('n', 2034), token('+', nil), token('(', nil), token('n', 3433), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil), token(Dhaka::END_SYMBOL_NAME, nil)], actual)
   end
 
   def test_an_input_with_unrecognized_characters
-    assert_raise(Dhaka::UnrecognizedInputCharacterException) {ArithmeticTokenizer.tokenize('2+(3 / (7 -& 5))')}
+    result = ArithmeticTokenizer.tokenize('2+(3 / (7 -& 5))')
+    assert(result.has_error?)
+    assert_equal(11, result.unexpected_char_index)
   end
 
   def test_another_input_with_illegal_characters
-    assert_raise(Dhaka::UnrecognizedInputCharacterException) {ArithmeticTokenizer.tokenize('2034 +(34b3 / (7 - 5))')}
+    result = ArithmeticTokenizer.tokenize('2034 +(34b3 / (7 - 5))')
+    assert(result.has_error?)
+    assert_equal(9, result.unexpected_char_index)
   end
 
   def token(symbol_name, value)
-    Dhaka::Token.new(ArithmeticGrammar.symbol_for_name(symbol_name), value)
+    Dhaka::Token.new(symbol_name, value ? value.to_s : nil, nil)
   end
 end

data/test/bracket_tokenizer.rb

@@ -8,7 +8,7 @@ class BracketTokenizer < Dhaka::Tokenizer
 
   for_state Dhaka::TOKENIZER_IDLE_STATE do
     for_characters(all_characters) do
-      tokens << Dhaka::Token.new(BracketGrammar.symbol_for_name(curr_char), nil)
+      create_token(curr_char, nil)
       advance
     end
   end

data/test/chittagong_driver_test.rb

@@ -0,0 +1,261 @@
+require "test/unit"
+require "yaml"
+require "chittagong_grammar"
+require "chittagong_tokenizer"
+require "chittagong_evaluator"
+require "fake_logger"
+
+unless Object.const_defined? :ChittagongParser
+  eval(Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new).compile_to_ruby_source_as(:ChittagongParser)) 
+end
+
+class ChittagongDriver
+  def marked_program(program, error_position)
+    if error_position > 0
+      program[0..(error_position-1)] + ">>>" + program[error_position..-1]
+    else
+      ">>>"+program
+    end
+  end
+  
+  def parse_error_message unexpected_token, program
+    if unexpected_token.symbol_name == Dhaka::END_SYMBOL_NAME
+      "Unexpected end of file."
+    else
+      "Unexpected token #{unexpected_token.symbol_name}:\n#{marked_program(program, unexpected_token.input_position)}"
+    end
+  end
+  
+  def tokenize_error_message unexpected_char_index, program
+    "Unexpected character #{program[unexpected_char_index].chr}:\n#{marked_program(program, unexpected_char_index)}"
+  end
+  
+  def evaluation_error_message evaluation_result, program
+    "#{evaluation_result.exception}:\n#{marked_program(program, evaluation_result.node.tokens[0].input_position)}"
+  end
+  
+  def run(program)
+    tokenize_result = ChittagongTokenizer.tokenize(program)
+    if tokenize_result.has_error?
+      return tokenize_error_message(tokenize_result.unexpected_char_index, program) 
+    end
+    
+    parse_result = ChittagongParser.parse(tokenize_result)
+    if parse_result.has_error?
+      return parse_error_message(parse_result.unexpected_token, program) 
+    end
+    
+    evaluation_result = ChittagongEvaluator.new([{}], output_stream = []).
+        evaluate(parse_result.parse_tree)
+    if evaluation_result.exception
+      return (output_stream << evaluation_error_message(evaluation_result, program)).
+        join("\n") 
+    end
+    
+    return output_stream.join("\n")
+  end
+  
+end
+
+
+class TestChittagongDriver < Test::Unit::TestCase
+  
+  def setup
+    @driver = ChittagongDriver.new
+  end
+  
+  def test_return_statement_not_allowed_in_main
+    # Programs with problems
+    program = "
+
+    if 1 > 2
+      return 5
+    end
+
+    print 2
+    "
+    assert_equal(
+    "Unexpected token return:
+
+
+    if 1 > 2
+      >>>return 5
+    end
+
+    print 2
+    ", @driver.run(program))
+  end
+  
+  def test_catches_unexpected_end_of_input
+    program = "
+      def"
+    assert_equal("Unexpected end of file.", @driver.run(program))
+  end
+  
+  def test_catches_unexpected_characters
+    program = "
+      print 2
+      def #}
+        "
+    assert_equal(
+    "Unexpected character #:
+
+      print 2
+      def >>>#}
+        ", @driver.run(program))
+  end
+  
+  def test_catches_undefined_variables
+    program = "
+      x = 1
+      y = 2
+      def foo(n)
+        return 2
+      end
+      foo(1)
+      
+      if x > y
+        print 999
+        print 777
+      else
+        print 66
+      end
+      
+      print x
+      print y
+      print 2*3+c
+      print 888
+      
+    "
+
+    assert_equal(
+    "66
+1
+2
+Undefined variable c:
+
+      x = 1
+      y = 2
+      def foo(n)
+        return 2
+      end
+      foo(1)
+      
+      if x > y
+        print 999
+        print 777
+      else
+        print 66
+      end
+      
+      print x
+      print y
+      print 2*3+>>>c
+      print 888
+      
+    ", @driver.run(program))
+  end
+
+  def test_catches_undefined_functions
+    program = "
+
+      def foo(n)
+        bar(x)
+      end
+      
+      foo(2)
+      
+    "
+    assert_equal(
+    "Undefined function bar:
+
+
+      def foo(n)
+        >>>bar(x)
+      end
+      
+      foo(2)
+      
+    ", @driver.run(program))
+  end
+  
+  def test_variable_scope
+    program = "
+      x = 1
+      
+      def bar(n)
+        print 999
+      end
+      
+      def foo(n)
+        bar(x)
+      end
+      
+      foo(2)
+      
+    "
+    assert_equal(
+    "Undefined variable x:
+
+      x = 1
+      
+      def bar(n)
+        print 999
+      end
+      
+      def foo(n)
+        bar(>>>x)
+      end
+      
+      foo(2)
+      
+    ", @driver.run(program))
+  end
+  
+  def test_nested_function_calls
+    
+    program = "
+    
+    def foo(a)
+      return a + 2
+    end
+    
+    def bar(b, c)
+      return b + c
+    end
+    
+    def baz(x, y, z)
+      return foo(y) + bar(x, z)
+    end
+    
+    print baz(foo(1), bar(2, 3), bar(6, 2))
+    
+    "
+    assert_equal("18", @driver.run(program))
+  end
+  
+  def test_wrong_number_of_arguments
+    
+    program = "
+    
+    def whatever(a, b)
+      return a + b
+    end
+    
+    whatever(1, 2, 3)
+    
+    "
+    assert_equal(
+    "Wrong number of arguments:
+
+    
+    def whatever(a, b)
+      return a + b
+    end
+    
+    whatever>>>(1, 2, 3)
+    
+    ", @driver.run(program))
+    
+  end
+end