dhaka 1.0.0 → 2.0.0

data/test/chittagong_parser_test.rb

@@ -6,42 +6,54 @@ require "fake_logger"
 
 class TestChittagongParser < Test::Unit::TestCase
   def setup
-    @parser = Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new)
+    fake_logger = FakeLogger.new
+    @parser = Dhaka::Parser.new(ChittagongGrammar, fake_logger)
+    assert_equal(80, fake_logger.warnings.size)
+    assert_equal(0, fake_logger.errors.size)
+    eval(@parser.compile_to_ruby_source_as(:ChittagongParser)) unless Module.const_defined? :ChittagongParser
   end
 
   def test_parses_a_series_of_statements
     token_stream = build_tokens(
       'newline',
-      'var_name', '=', 'int_literal', 'newline',
-      'print', 'var_name', 'newline',
+      'word_literal', '=', 'int_literal', 'newline',
+      'print', 'word_literal', 'newline',
       'newline',
-      'var_name', '=', 'var_name', 'newline',
-      'newline'
+      'word_literal', '=', 'word_literal', 'newline',
+      'newline', Dhaka::END_SYMBOL_NAME
     )
     
     result = @parser.parse(token_stream)
     
     assert_equal(["single_term",
      "some_terms",
+     "variable_name",
      "literal",
-     "assignment_expression",
-     "single_statement",
+     "assignment_statement",
+     "main_body_simple_statement",
+     "single_main_body_statement",
      "single_term",
+     "variable_name",
      "variable_reference",
      "print_statement",
-     "multiple_statements",
+     "main_body_simple_statement",
+     "multiple_main_body_statements",
      "single_term",
      "multiple_terms",
+     "variable_name",
+     "variable_name",
      "variable_reference",
-     "assignment_expression",
-     "multiple_statements",
+     "assignment_statement",
+     "main_body_simple_statement",
+     "multiple_main_body_statements",
      "single_term",
      "multiple_terms",
      "some_terms",
-     "program"], result.syntax_tree.linearize)
+     "program"], result.parse_tree.linearize.collect {|node| node.production.name})
+     
   end
 
   def build_tokens *symbol_names
-    symbol_names.collect {|symbol_name| Dhaka::Token.new(ChittagongGrammar.symbol_for_name(symbol_name), nil)}
+    symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
   end
 end 

data/test/chittagong_test.rb

@@ -5,14 +5,28 @@ require "chittagong_tokenizer"
 require "chittagong_evaluator"
 require "fake_logger"
 
+unless Object.const_defined? :ChittagongParser
+  eval(Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new).compile_to_ruby_source_as(:ChittagongParser)) 
+end
+
+
 class TestChittagong < Test::Unit::TestCase
   
-  def setup
-    @parser = Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new)
+  def fact(n)
+    return 1 if n==1
+    n * fact(n-1)
+  end
+  
+  def program_output program
+    output_stream = []
+    parse_result = ChittagongParser.parse(ChittagongTokenizer.tokenize(program))
+    result = ChittagongEvaluator.new([{}], output_stream).evaluate(parse_result.parse_tree)
+    return result, output_stream
   end
   
-  def test_tokenizes_parses_and_evaluates_a_small_program
+  def test_iterative_fibonacci_without_functions
     program = "
+
     n = 1
     a = 0
     b = 1
@@ -25,9 +39,137 @@ class TestChittagong < Test::Unit::TestCase
     end
     
     "
-    symbol_table = {}
-    output_stream = []
-    ChittagongEvaluator.new(symbol_table, output_stream).evaluate(@parser.parse(ChittagongTokenizer.tokenize(program)).syntax_tree)
+
+    result, output_stream = program_output(program)
+    assert_equal(["1", "1", "2", "3", "5", "8", "13", "21", "34"], output_stream)
+  end
+
+  def test_iterative_fibonacci_with_functions
+    program = "
+
+    def fib(n)
+      i = 0
+      a = 0
+      b = 1
+      while i < n
+        c = a
+        a = b
+        b = c + b
+        i = i + 1
+      end
+      return b
+    end
+    
+    x = 0
+    while x < 9
+      print fib(x)
+      x = x + 1
+    end
+    
+    "
+    result, output_stream = program_output(program)
+    assert_equal(["1", "1", "2", "3", "5", "8", "13", "21", "34"], output_stream)
+  end
+  
+  def test_recursive_fibonacci
+    program = "
+
+    def fib(n)
+      if n == 0
+        return 1
+      end
+      if n == -1
+        return 0
+      end
+      return fib(n-1) + fib(n-2)
+    end
+    
+    x = 0
+    while x < 9
+      print fib(x)
+      x = x + 1
+    end
+    
+    "
+    result, output_stream = program_output(program)
     assert_equal(["1", "1", "2", "3", "5", "8", "13", "21", "34"], output_stream)
   end
+  
+  def test_recursive_factorial
+    program = "
+    def fact(n)
+      if n == 1 
+        return 1
+      end
+      return n * fact(n-1)
+    end
+
+    n = 1
+    while n < 11
+      print fact(n)
+      n = n+1
+    end"
+    
+    result, output_stream = program_output(program)
+    assert_equal((1..10).collect {|i| fact(i).to_s}, output_stream)
+  end
+  
+  def test_various_things
+    program = "
+    
+    a = 1
+    b = 2
+    c = 3
+    
+    def foo(a, b, c)
+      print a
+      print b
+      print c
+      return c
+      print 999
+    end
+    
+    foo(4, a, 6)
+    
+    "
+   
+    result, output_stream = program_output(program)
+    assert_equal(["4", "1", "6"], output_stream)
+  end
+  
+  def test_if_else_block
+    program = "
+
+    def foo(a, b)
+      if a < b
+        print 1
+      else
+        print 2
+      end
+    end
+
+    foo(1, 2)
+    foo(2, 1)
+
+    "
+
+    result, output_stream = program_output(program)
+    assert_equal(["1", "2"], output_stream)
+  end
+  
+  def test_no_arg_functions
+    program = "
+
+    def foo()
+      print 1
+      print 2
+    end
+    foo()
+
+    "
+
+    result, output_stream = program_output(program)
+    assert_equal(["1", "2"], output_stream)
+  end
+
 end

data/test/chittagong_tokenizer.rb

@@ -4,32 +4,39 @@ require 'chittagong_grammar'
 
 class ChittagongTokenizer < Dhaka::Tokenizer
 
-  KEYWORDS = ['print', 'if', 'else', 'end', 'while']
+  KEYWORDS = ['print', 'if', 'else', 'end', 'while', 'def', 'return']
   
   digits = ('0'..'9').to_a
   letters = ('a'..'z').to_a
   parenths = ['(', ')']
-  operators = ['-', '+', '/', '*', '^', '!', '>', '<', '=']
+  operators = ['-', '+', '/', '*', '^', '!', '>', '<']
+  equal_sign = ['=']
   whitespace = [' ']
+  arg_separator = [',']
   newline = ["\n"]
   
-  all_characters = digits + letters + parenths + operators + whitespace + newline
+  all_characters = digits + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
 
   for_state Dhaka::TOKENIZER_IDLE_STATE do
-    for_characters(all_characters - (digits + letters + newline + whitespace)) do
-      tokens << Dhaka::Token.new(ChittagongGrammar.symbol_for_name(curr_char), nil)
+    for_characters(all_characters - (digits + letters + newline + whitespace + equal_sign)) do
+      create_token(curr_char, nil)
       advance
     end
+    for_characters(equal_sign) do
+      create_token('=', nil)
+      advance
+      switch_to :get_equal_sign_operator
+    end
     for_characters digits do
-      self.accumulator = ''
+      create_token('int_literal', '')
       switch_to :get_integer_literal
     end
     for_characters letters do
-      self.accumulator = ''
+      create_token(nil, '')
       switch_to :get_word_literal
     end
     for_character newline do
-      tokens << Dhaka::Token.new(ChittagongGrammar.symbol_for_name('newline'), nil)
+      create_token('newline', nil)
       advance
     end
     for_character whitespace do
@@ -37,40 +44,45 @@ class ChittagongTokenizer < Dhaka::Tokenizer
     end
   end
   
+  for_state :get_equal_sign_operator do
+    for_characters all_characters - equal_sign do
+      switch_to Dhaka::TOKENIZER_IDLE_STATE
+    end
+    for_character equal_sign do
+      curr_token.symbol_name += '='
+      advance
+      switch_to Dhaka::TOKENIZER_IDLE_STATE
+    end
+  end
+  
   for_state :get_integer_literal do
     for_characters all_characters - digits do
-      tokens << integer_literal_token(accumulator.to_i)
       switch_to Dhaka::TOKENIZER_IDLE_STATE
     end
     for_characters digits do
-      self.accumulator += curr_char
+      curr_token.value += curr_char
       advance
-      tokens << integer_literal_token(accumulator.to_i) unless curr_char
     end
   end
 
   for_state :get_word_literal do
     for_characters all_characters - letters do
-      tokens << word_literal_token(accumulator)
+      curr_token.symbol_name = word_literal_symbol(curr_token.value)
       switch_to Dhaka::TOKENIZER_IDLE_STATE
     end
     for_characters letters do
-      self.accumulator += curr_char
+      curr_token.value += curr_char
       advance
-      tokens << word_literal_token(accumulator) unless curr_char
+      curr_token.symbol_name = word_literal_symbol(curr_token.value) unless curr_char
     end
   end
   
-  def word_literal_token(value)
+  def word_literal_symbol(value)
     if KEYWORDS.include? value
-      Dhaka::Token.new(ChittagongGrammar.symbol_for_name(value), nil)
+      value
     else
-      Dhaka::Token.new(ChittagongGrammar.symbol_for_name('var_name'), value)
+      'word_literal'
     end
   end
   
-  def integer_literal_token(value)
-    Dhaka::Token.new(ChittagongGrammar.symbol_for_name('int_literal'), value)
-  end
-  
 end

data/test/chittagong_tokenizer_test.rb

@@ -7,6 +7,7 @@ class TestChittagongTokenizer < Test::Unit::TestCase
     input = "
     x = 2 * 4
     y = 2 * x
+    v = x == y
     if x > y
       print x
     else
@@ -14,32 +15,39 @@ class TestChittagongTokenizer < Test::Unit::TestCase
     end
     "
     assert_equal(["newline",
-     "var_name",
+     "word_literal",
      "=",
      "int_literal",
      "*",
      "int_literal",
      "newline",
-     "var_name",
+     "word_literal",
      "=",
      "int_literal",
      "*",
-     "var_name",
+     "word_literal",
+     "newline",
+     "word_literal",
+     "=",
+     "word_literal",
+     "==",
+     "word_literal",
      "newline",
      "if",
-     "var_name",
+     "word_literal",
      ">",
-     "var_name",
+     "word_literal",
      "newline",
      "print",
-     "var_name",
+     "word_literal",
      "newline",
      "else",
      "newline",
      "print",
-     "var_name",
+     "word_literal",
      "newline",
      "end",
-     "newline"], ChittagongTokenizer.tokenize(input).collect {|token| token.grammar_symbol.name})
+     "newline",
+     Dhaka::END_SYMBOL_NAME], ChittagongTokenizer.tokenize(input).collect {|token| token.symbol_name})
   end
 end

data/test/compiled_parser_test.rb

@@ -9,8 +9,8 @@ eval(Dhaka::Parser.new(ArithmeticGrammar).compile_to_ruby_source_as('CompiledAri
 class TestCompiledParser < Test::Unit::TestCase
   include ArithmeticTestMethods
   
-  def test_compiled_parser_generates_syntax_tree_for_simple_grammar
-    syntax_tree = Foo.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#'], Foo.grammar)).syntax_tree
+  def test_compiled_parser_generates_parse_tree_for_simple_grammar
+    parse_tree = Foo.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME])).parse_tree
     assert_equal \
       ["literal",
        "term",
@@ -25,11 +25,11 @@ class TestCompiledParser < Test::Unit::TestCase
        "literal",
        "subtraction",
        "expression",
-       "start"], syntax_tree.linearize
+       "start"], parse_tree.linearize.collect {|node| node.production.name}
   end
 
-  def test_compiled_parser_generates_syntax_tree_for_arithmetic_grammar
-    parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')']
+  def test_compiled_parser_generates_parse_tree_for_arithmetic_grammar
+    parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')',Dhaka::END_SYMBOL_NAME]
     assert_equal \
       ["getting_literals",
        "factor",
@@ -50,21 +50,23 @@ class TestCompiledParser < Test::Unit::TestCase
        "unpacking_parenthetized_expression",
        "factor",
        "term",
-       "expression"], parse(build_tokens(parser_input, CompiledArithmeticParser.grammar)).linearize
+       "expression"], parse(build_tokens(parser_input)).linearize.collect {|node| node.production.name}
   end
   
-  def test_parse_result_has_error_if_empty_token_array
-    assert CompiledArithmeticParser.parse([]).has_error?
+  def test_parse_result_has_nil_parse_tree_if_empty_token_array
+    parse_result = CompiledArithmeticParser.parse([])
+    assert !CompiledArithmeticParser.parse([]).has_error?
+    assert_nil parse_result.parse_tree
   end
 
   def test_parser_returns_error_result_with_index_of_bad_token_if_parse_error
-    parse_result = CompiledArithmeticParser.parse(build_tokens(['(', '-', ')'], CompiledArithmeticParser.grammar))
+    parse_result = CompiledArithmeticParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
     assert parse_result.has_error?
-    assert_equal 1, parse_result.bad_token_index
+    assert_equal '-', parse_result.unexpected_token.symbol_name
   end
 
-  def build_tokens(token_symbol_names, grammar)
-    token_symbol_names.collect {|symbol_name| Dhaka::Token.new(grammar.symbol_for_name(symbol_name), nil)}
+  def build_tokens(token_symbol_names)
+    token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
   end
 
 end