dhaka 0.0.6 → 1.0.0

data/lib/evaluator/evaluator.rb

@@ -13,31 +13,24 @@ module Dhaka
 
     # Instantiates a new evaluator with the syntax tree of a parsed expression. Only subclasses
     # of Evaluator are directly instantiated.
-    def initialize(syntax_tree)
-      @syntax_tree = syntax_tree
-      @node_stack = []
-    end
-
-    # Returns the evaluation result.
-    def result
-      evaluate(@syntax_tree)
-    end
-
-    private
 
-    def child_nodes
-      @node_stack[-1]
-    end
-  
+    # Evaluate a syntax tree node.
     def evaluate node
-      return node if (ParseTreeLeafNode === node)
-      @node_stack << node.child_nodes.collect {|child_node| evaluate(child_node)}
+      @node_stack ||= []
+      @node_stack << node.child_nodes
       proc = self.class.actions[node.production.name]
       result = self.instance_eval(&proc)
       @node_stack.pop
       result
     end
 
+    # Returns the array of child nodes of the node being currently evaluated. 
+    def child_nodes
+      @node_stack[-1]
+    end
+
+    private
+  
     def self.inherited(evaluator)
       class << evaluator
         attr_accessor :grammar, :actions
@@ -45,7 +38,7 @@ module Dhaka
     end
   
     def self.define_evaluation_rules
-      default_action = Proc.new { child_nodes[0] }
+      default_action = Proc.new { evaluate(child_nodes[0]) }
       self.actions = Hash.new { |hash, key| default_action }
       yield
       check_definitions

data/lib/tokenizer/tokenizer.rb

@@ -12,7 +12,7 @@ module Dhaka
       @char_index = char_index
     end
     def to_s
-      "Unrecognized character #{input[char_index].chr} encountered while tokenizing:\n #{input}"
+      "Unrecognized character '#{input[char_index].chr}' encountered while tokenizing:\n #{input} at index #{char_index}"
     end
   end
 

data/test/all_tests.rb

@@ -11,4 +11,8 @@ require 'malformed_grammar_test'
 require 'brackets_test'
 require 'arithmetic_precedence_grammar_test'
 require 'arithmetic_precedence_parser_test'
-require 'precedence_grammar_test'
+require 'precedence_grammar_test'
+require 'chittagong_parser_test'
+require 'chittagong_evaluator_test'
+require 'chittagong_tokenizer_test'
+require 'chittagong_test'

data/test/arithmetic_evaluator.rb

@@ -8,19 +8,19 @@ class ArithmeticEvaluator < Dhaka::Evaluator
   define_evaluation_rules do
     
     for_subtraction do 
-      child_nodes[0] - child_nodes[2]
+      evaluate(child_nodes[0]) - evaluate(child_nodes[2])
     end
 
     for_addition do
-      child_nodes[0] + child_nodes[2]
+      evaluate(child_nodes[0]) + evaluate(child_nodes[2])
     end
 
     for_division do
-      child_nodes[0].to_f/child_nodes[2] 
+      evaluate(child_nodes[0]).to_f/evaluate(child_nodes[2]) 
     end
 
     for_multiplication do
-      child_nodes[0] * child_nodes[2]
+      evaluate(child_nodes[0]) * evaluate(child_nodes[2])
     end
 
     for_getting_literals do
@@ -28,7 +28,7 @@ class ArithmeticEvaluator < Dhaka::Evaluator
     end
 
     for_unpacking_parenthetized_expression do
-      child_nodes[1]
+      evaluate(child_nodes[1])
     end
     
     for_empty_args do
@@ -36,15 +36,15 @@ class ArithmeticEvaluator < Dhaka::Evaluator
     end
 
     for_evaluating_function do
-      child_nodes[0].call child_nodes[2]
+      evaluate(child_nodes[0]).call evaluate(child_nodes[2])
     end
   
     for_concatenating_args do
-      [child_nodes[0]]+child_nodes[2]
+      [evaluate(child_nodes[0])]+evaluate(child_nodes[2])
     end
   
     for_single_args do
-      [child_nodes[0]]
+      [evaluate(child_nodes[0])]
     end
   
     for_min_function do
@@ -57,10 +57,9 @@ class ArithmeticEvaluator < Dhaka::Evaluator
 
   end
   
-  def initialize(syntax_tree, min_function, max_function)
+  def initialize(min_function, max_function)
     @min_function = min_function
     @max_function = max_function
-    super(syntax_tree)
   end
 
 end

data/test/arithmetic_evaluator_test.rb

@@ -15,7 +15,7 @@ class TestArithmeticEvaluator < Test::Unit::TestCase
     
     token_stream = [token('n', 2), token('-', nil), token('n', 4)]
     syntax_tree = parse(token_stream)
-    assert_equal -2, ArithmeticEvaluator.new(syntax_tree, @min_func, @max_func).result
+    assert_equal -2, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
     
   end
 
@@ -23,7 +23,7 @@ class TestArithmeticEvaluator < Test::Unit::TestCase
 
     token_stream = [token('n', 2), token('-', nil), token('(', nil), token('n', 3), token('/', nil), token('n', 4), token(')', nil)]
     syntax_tree = parse(token_stream)
-    assert_equal 1.25, ArithmeticEvaluator.new(syntax_tree, @min_func, @max_func).result
+    assert_equal 1.25, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
 
   end
 
@@ -31,7 +31,7 @@ class TestArithmeticEvaluator < Test::Unit::TestCase
 
     token_stream = [token('n', 2), token('+', nil), token('(', nil), token('n', 3), token('/', nil), token('(', nil), token('n', 7), token('-', nil), token('n', 5), token(')', nil) , token(')', nil)]
     syntax_tree = parse(token_stream)
-    assert_equal 3.5, ArithmeticEvaluator.new(syntax_tree, @min_func, @max_func).result
+    assert_equal 3.5, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
 
   end
 
@@ -39,7 +39,7 @@ class TestArithmeticEvaluator < Test::Unit::TestCase
 
     token_stream = [token('n', 2), token('+', nil), token('h', nil), token('(', nil), token('n', 3), token(',', nil), token('n', 4), token(')', nil)]
     syntax_tree = parse(token_stream)
-    assert_equal 6, ArithmeticEvaluator.new(syntax_tree, @min_func, @max_func).result
+    assert_equal 6, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
 
   end
 
@@ -47,7 +47,7 @@ class TestArithmeticEvaluator < Test::Unit::TestCase
 
     token_stream = [token('n', 2), token('+', nil), token('l', nil), token('(', nil), token('n', 3), token(',', nil), token('n', 4), token(')', nil)]
     syntax_tree = parse(token_stream)
-    assert_equal 5, ArithmeticEvaluator.new(syntax_tree, @min_func, @max_func).result
+    assert_equal 5, ArithmeticEvaluator.new(@min_func, @max_func).evaluate(syntax_tree)
 
   end
 

data/test/arithmetic_precedence_evaluator.rb

@@ -8,19 +8,19 @@ class ArithmeticPrecedenceEvaluator < Dhaka::Evaluator
   define_evaluation_rules do
     
     for_subtraction do 
-      child_nodes[0] - child_nodes[2]
+      evaluate(child_nodes[0]) - evaluate(child_nodes[2])
     end
 
     for_addition do
-      child_nodes[0] + child_nodes[2]
+      evaluate(child_nodes[0]) + evaluate(child_nodes[2])
     end
 
     for_division do
-      child_nodes[0].to_f/child_nodes[2] 
+      evaluate(child_nodes[0]).to_f/evaluate(child_nodes[2]) 
     end
 
     for_multiplication do
-      child_nodes[0] * child_nodes[2]
+      evaluate(child_nodes[0]) * evaluate(child_nodes[2])
     end
 
     for_literal do
@@ -28,15 +28,15 @@ class ArithmeticPrecedenceEvaluator < Dhaka::Evaluator
     end
 
     for_parenthetized_expression do
-      child_nodes[1]
+      evaluate(child_nodes[1])
     end
     
     for_negated_expression do
-      -child_nodes[1]
+      -evaluate(child_nodes[1])
     end
     
     for_power do
-      child_nodes[0]**child_nodes[2]
+      evaluate(child_nodes[0])**evaluate(child_nodes[2])
     end
 
   end

data/test/arithmetic_precedence_parser_test.rb

@@ -26,7 +26,7 @@ class TestArithmeticPrecedenceParser < Test::Unit::TestCase
   end
 
   def evaluate(syntax_tree)
-    ArithmeticPrecedenceEvaluator.new(syntax_tree).result
+    ArithmeticPrecedenceEvaluator.new.evaluate(syntax_tree)
   end
 
 end

data/test/chittagong_evaluator.rb

@@ -0,0 +1,107 @@
+require File.dirname(__FILE__)+'/../lib/dhaka'
+require 'chittagong_grammar'
+
+class ChittagongEvaluator < Dhaka::Evaluator
+
+  self.grammar = ChittagongGrammar
+
+  define_evaluation_rules do
+    ['no_terms', 'multiple_terms', 'single_term', 'some_terms'].each {|production_name| eval("for_#{production_name} do end")}
+
+    for_program do
+      evaluate(child_nodes[1])
+    end
+    
+    for_multiple_statements do
+      evaluate(child_nodes[0])
+      evaluate(child_nodes[2])
+    end
+    
+    for_print_statement do
+      @output_stream << evaluate(child_nodes[1]).to_s
+    end
+
+    for_subtraction do 
+      evaluate(child_nodes[0]) - evaluate(child_nodes[2])
+    end
+
+    for_addition do
+      evaluate(child_nodes[0]) + evaluate(child_nodes[2])
+    end
+
+    for_division do
+      evaluate(child_nodes[0]).to_f/evaluate(child_nodes[2])
+    end
+
+    for_multiplication do
+      evaluate(child_nodes[0]) * evaluate(child_nodes[2])
+    end
+
+    for_literal do
+      child_nodes[0].token.value
+    end
+
+    for_parenthetized_expression do
+      evaluate(child_nodes[1])
+    end
+    
+    for_negated_expression do
+      -evaluate(child_nodes[1])
+    end
+    
+    for_power do
+      evaluate(child_nodes[0])**evaluate(child_nodes[2])
+    end
+    
+    for_negation do
+      !evaluate(child_nodes[1])
+    end
+    
+    for_if_else_statement do
+      if evaluate(child_nodes[1])
+        evaluate(child_nodes[3])
+      else
+        evaluate(child_nodes[7])
+      end
+    end
+    
+    for_if_statement do
+      if evaluate(child_nodes[1])
+        evaluate(child_nodes[3])
+      end
+    end
+    
+    for_while_statement do
+      while evaluate(child_nodes[1])
+        evaluate(child_nodes[3])
+      end
+    end
+    
+    for_less_than_comparison do
+      evaluate(child_nodes[0]) < evaluate(child_nodes[2])
+    end
+
+    for_greater_than_comparison do
+      evaluate(child_nodes[0]) > evaluate(child_nodes[2])
+    end
+
+    for_equality_comparison do
+      evaluate(child_nodes[0]) == evaluate(child_nodes[2])
+    end
+        
+    for_variable_reference do
+      @symbol_table[child_nodes[0].token.value]
+    end
+    
+    for_assignment_expression do
+      @symbol_table[child_nodes[0].token.value] = evaluate(child_nodes[2])
+    end
+    
+    def initialize(symbol_table, output_stream)
+      @symbol_table = symbol_table
+      @output_stream = output_stream
+    end
+
+  end
+
+end

data/test/chittagong_evaluator_test.rb

@@ -0,0 +1,41 @@
+require "test/unit"
+
+require "chittagong_evaluator"
+require "fake_logger"
+
+class TestChittagongEvaluator < Test::Unit::TestCase
+
+  def setup
+    @logger = FakeLogger.new
+    @parser = Dhaka::Parser.new(ChittagongGrammar, @logger)
+  end
+
+  def test_evaluates_a_simple_program
+    assert_equal(30, @logger.warnings.size)
+    assert_equal(0, @logger.errors.size)
+    token_stream = build_tokens(
+      ['newline'],
+      ['var_name', 'x'], ['='], ['int_literal', 23], ['newline'],
+      ['print'], ['var_name', 'x'], ['newline'],
+      ['newline'],
+      ['var_name', 'y'], ['='], ['var_name', 'x'], ['+'], ['var_name', 'x'], ['newline'],
+      ['print'], ['var_name', 'y'], ['*'], ['var_name', 'x'], ['newline'],
+      ['if'], ['var_name', 'x'], ['>'], ['var_name', 'y'], ['newline'],
+      ['print'], ['var_name', 'x'], ['newline'],
+      ['else'], ['newline'],
+      ['print'], ['var_name', 'y'], ['newline'],
+      ['end'], ['newline'],
+      ['newline']
+    )
+    symbol_table = {}
+    output_stream = []
+    ChittagongEvaluator.new(symbol_table, output_stream).evaluate(@parser.parse(token_stream).syntax_tree)
+    assert_equal(23, symbol_table['x'])
+    assert_equal(46, symbol_table['y'])
+    assert_equal(['23', '1058', '46'], output_stream)
+  end
+  
+  def build_tokens *tokens
+    tokens.collect {|token| Dhaka::Token.new(ChittagongGrammar.symbol_for_name(token[0]), token[1])}
+  end
+end

data/test/chittagong_grammar.rb

@@ -0,0 +1,58 @@
+require File.dirname(__FILE__)+'/../lib/dhaka'
+
+class ChittagongGrammar < Dhaka::Grammar
+
+  precedences do
+    left ['+', '-']
+    left ['*', '/']
+    nonassoc ['^']
+  end
+  
+  for_symbol(Dhaka::START_SYMBOL_NAME) do
+    program ['opt_terms', 'statements', 'opt_terms']
+  end
+
+  for_symbol('statements') do
+    single_statement ['statement']
+    multiple_statements ['statements', 'terms', 'statement']
+  end
+
+  for_symbol('statement') do
+    assignment_expression ['var_name', '=', 'expression']
+    print_statement ['print', 'expression']
+    if_statement ['if', 'boolean_expression', 'terms', 'statements', 'terms', 'end']
+    if_else_statement ['if', 'boolean_expression', 'terms', 'statements', 'terms', 'else', 'terms', 'statements', 'terms', 'end']
+    while_statement ['while', 'boolean_expression', 'terms', 'statements', 'terms', 'end']
+  end
+  
+  for_symbol('boolean_expression') do
+    negation ['!', 'boolean_expression']
+    equality_comparison ['expression', '=', '=', 'expression']
+    greater_than_comparison ['expression', '>', 'expression']
+    less_than_comparison ['expression', '<', 'expression']
+  end
+  
+
+  for_symbol('expression') do
+    addition ['expression', '+', 'expression']
+    subtraction ['expression', '-', 'expression']
+    multiplication ['expression', '*', 'expression']
+    division ['expression', '/', 'expression']
+    power ['expression', '^', 'expression']
+    literal ['int_literal']
+    variable_reference ['var_name']
+    parenthetized_expression ['(', 'expression', ')']
+    negated_expression ['-', 'expression'], :prec => '*'
+  end
+  
+  for_symbol('terms') do
+    single_term ['newline']
+    multiple_terms ['terms', 'newline']
+  end
+
+  for_symbol('opt_terms') do
+    some_terms ['terms']
+    no_terms []
+  end
+  
+end

data/test/chittagong_parser_test.rb

@@ -0,0 +1,47 @@
+require File.dirname(__FILE__)+'/../lib/dhaka'
+
+require 'chittagong_grammar'
+require "test/unit"
+require "fake_logger"
+
+class TestChittagongParser < Test::Unit::TestCase
+  def setup
+    @parser = Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new)
+  end
+
+  def test_parses_a_series_of_statements
+    token_stream = build_tokens(
+      'newline',
+      'var_name', '=', 'int_literal', 'newline',
+      'print', 'var_name', 'newline',
+      'newline',
+      'var_name', '=', 'var_name', 'newline',
+      'newline'
+    )
+    
+    result = @parser.parse(token_stream)
+    
+    assert_equal(["single_term",
+     "some_terms",
+     "literal",
+     "assignment_expression",
+     "single_statement",
+     "single_term",
+     "variable_reference",
+     "print_statement",
+     "multiple_statements",
+     "single_term",
+     "multiple_terms",
+     "variable_reference",
+     "assignment_expression",
+     "multiple_statements",
+     "single_term",
+     "multiple_terms",
+     "some_terms",
+     "program"], result.syntax_tree.linearize)
+  end
+
+  def build_tokens *symbol_names
+    symbol_names.collect {|symbol_name| Dhaka::Token.new(ChittagongGrammar.symbol_for_name(symbol_name), nil)}
+  end
+end 

data/test/chittagong_test.rb

@@ -0,0 +1,33 @@
+require "test/unit"
+
+require "chittagong_grammar"
+require "chittagong_tokenizer"
+require "chittagong_evaluator"
+require "fake_logger"
+
+class TestChittagong < Test::Unit::TestCase
+  
+  def setup
+    @parser = Dhaka::Parser.new(ChittagongGrammar, FakeLogger.new)
+  end
+  
+  def test_tokenizes_parses_and_evaluates_a_small_program
+    program = "
+    n = 1
+    a = 0
+    b = 1
+    while n < 10
+      print b
+      c = a
+      a = b
+      b = c + b
+      n = n + 1
+    end
+    
+    "
+    symbol_table = {}
+    output_stream = []
+    ChittagongEvaluator.new(symbol_table, output_stream).evaluate(@parser.parse(ChittagongTokenizer.tokenize(program)).syntax_tree)
+    assert_equal(["1", "1", "2", "3", "5", "8", "13", "21", "34"], output_stream)
+  end
+end

data/test/chittagong_tokenizer.rb

@@ -0,0 +1,76 @@
+require File.dirname(__FILE__)+'/../lib/dhaka'
+require 'chittagong_grammar'
+
+
+class ChittagongTokenizer < Dhaka::Tokenizer
+
+  KEYWORDS = ['print', 'if', 'else', 'end', 'while']
+  
+  digits = ('0'..'9').to_a
+  letters = ('a'..'z').to_a
+  parenths = ['(', ')']
+  operators = ['-', '+', '/', '*', '^', '!', '>', '<', '=']
+  whitespace = [' ']
+  newline = ["\n"]
+  
+  all_characters = digits + letters + parenths + operators + whitespace + newline
+
+  for_state Dhaka::TOKENIZER_IDLE_STATE do
+    for_characters(all_characters - (digits + letters + newline + whitespace)) do
+      tokens << Dhaka::Token.new(ChittagongGrammar.symbol_for_name(curr_char), nil)
+      advance
+    end
+    for_characters digits do
+      self.accumulator = ''
+      switch_to :get_integer_literal
+    end
+    for_characters letters do
+      self.accumulator = ''
+      switch_to :get_word_literal
+    end
+    for_character newline do
+      tokens << Dhaka::Token.new(ChittagongGrammar.symbol_for_name('newline'), nil)
+      advance
+    end
+    for_character whitespace do
+      advance
+    end
+  end
+  
+  for_state :get_integer_literal do
+    for_characters all_characters - digits do
+      tokens << integer_literal_token(accumulator.to_i)
+      switch_to Dhaka::TOKENIZER_IDLE_STATE
+    end
+    for_characters digits do
+      self.accumulator += curr_char
+      advance
+      tokens << integer_literal_token(accumulator.to_i) unless curr_char
+    end
+  end
+
+  for_state :get_word_literal do
+    for_characters all_characters - letters do
+      tokens << word_literal_token(accumulator)
+      switch_to Dhaka::TOKENIZER_IDLE_STATE
+    end
+    for_characters letters do
+      self.accumulator += curr_char
+      advance
+      tokens << word_literal_token(accumulator) unless curr_char
+    end
+  end
+  
+  def word_literal_token(value)
+    if KEYWORDS.include? value
+      Dhaka::Token.new(ChittagongGrammar.symbol_for_name(value), nil)
+    else
+      Dhaka::Token.new(ChittagongGrammar.symbol_for_name('var_name'), value)
+    end
+  end
+  
+  def integer_literal_token(value)
+    Dhaka::Token.new(ChittagongGrammar.symbol_for_name('int_literal'), value)
+  end
+  
+end

data/test/chittagong_tokenizer_test.rb

@@ -0,0 +1,45 @@
+require "test/unit"
+
+require "chittagong_tokenizer"
+
+class TestChittagongTokenizer < Test::Unit::TestCase
+  def test_tokenizes_a_program
+    input = "
+    x = 2 * 4
+    y = 2 * x
+    if x > y
+      print x
+    else
+      print y
+    end
+    "
+    assert_equal(["newline",
+     "var_name",
+     "=",
+     "int_literal",
+     "*",
+     "int_literal",
+     "newline",
+     "var_name",
+     "=",
+     "int_literal",
+     "*",
+     "var_name",
+     "newline",
+     "if",
+     "var_name",
+     ">",
+     "var_name",
+     "newline",
+     "print",
+     "var_name",
+     "newline",
+     "else",
+     "newline",
+     "print",
+     "var_name",
+     "newline",
+     "end",
+     "newline"], ChittagongTokenizer.tokenize(input).collect {|token| token.grammar_symbol.name})
+  end
+end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: dhaka
 version: !ruby/object:Gem::Version 
-  version: 0.0.6
-date: 2007-01-07 00:00:00 -05:00
+  version: 1.0.0
+date: 2007-01-08 00:00:00 -05:00
 summary: An LALR1 parser generator written in Ruby
 require_paths: 
 - lib
@@ -65,6 +65,13 @@ files:
 - test/bracket_grammar.rb
 - test/bracket_tokenizer.rb
 - test/brackets_test.rb
+- test/chittagong_evaluator.rb
+- test/chittagong_evaluator_test.rb
+- test/chittagong_grammar.rb
+- test/chittagong_parser_test.rb
+- test/chittagong_test.rb
+- test/chittagong_tokenizer.rb
+- test/chittagong_tokenizer_test.rb
 - test/compiled_parser_test.rb
 - test/evaluator_test.rb
 - test/fake_logger.rb