dhaka 2.0.0 → 2.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (68) hide show
  1. data/lib/dhaka.rb +24 -22
  2. data/lib/evaluator/evaluator.rb +42 -44
  3. data/lib/grammar/closure_hash.rb +4 -3
  4. data/lib/grammar/grammar.rb +113 -110
  5. data/lib/grammar/grammar_symbol.rb +6 -3
  6. data/lib/grammar/precedence.rb +3 -2
  7. data/lib/grammar/production.rb +5 -6
  8. data/lib/parser/action.rb +16 -11
  9. data/lib/parser/channel.rb +22 -16
  10. data/lib/parser/compiled_parser.rb +28 -22
  11. data/lib/parser/conflict.rb +54 -0
  12. data/lib/parser/item.rb +19 -19
  13. data/lib/parser/parse_result.rb +16 -1
  14. data/lib/parser/parse_tree.rb +15 -9
  15. data/lib/parser/parser.rb +51 -80
  16. data/lib/parser/parser_run.rb +6 -6
  17. data/lib/parser/parser_state.rb +16 -18
  18. data/lib/parser/token.rb +6 -4
  19. data/lib/tokenizer/tokenizer.rb +34 -31
  20. data/test/all_tests.rb +4 -18
  21. data/test/another_lalr_but_not_slr_grammar.rb +9 -5
  22. data/test/{arithmetic_evaluator.rb → arithmetic/arithmetic_evaluator.rb} +1 -2
  23. data/test/{arithmetic_evaluator_test.rb → arithmetic/arithmetic_evaluator_test.rb} +9 -20
  24. data/test/arithmetic/arithmetic_grammar.rb +41 -0
  25. data/test/{arithmetic_grammar_test.rb → arithmetic/arithmetic_grammar_test.rb} +2 -4
  26. data/test/{arithmetic_test_methods.rb → arithmetic/arithmetic_test_methods.rb} +1 -3
  27. data/test/{arithmetic_tokenizer.rb → arithmetic/arithmetic_tokenizer.rb} +8 -10
  28. data/test/{arithmetic_tokenizer_test.rb → arithmetic/arithmetic_tokenizer_test.rb} +4 -2
  29. data/test/{arithmetic_precedence_evaluator.rb → arithmetic_precedence/arithmetic_precedence_evaluator.rb} +1 -2
  30. data/test/arithmetic_precedence/arithmetic_precedence_grammar.rb +24 -0
  31. data/test/{arithmetic_precedence_grammar_test.rb → arithmetic_precedence/arithmetic_precedence_grammar_test.rb} +2 -3
  32. data/test/arithmetic_precedence/arithmetic_precedence_parser_test.rb +31 -0
  33. data/test/{arithmetic_precedence_tokenizer.rb → arithmetic_precedence/arithmetic_precedence_tokenizer.rb} +8 -10
  34. data/test/brackets/bracket_grammar.rb +23 -0
  35. data/test/{bracket_tokenizer.rb → brackets/bracket_tokenizer.rb} +2 -4
  36. data/test/{brackets_test.rb → brackets/brackets_test.rb} +3 -4
  37. data/test/chittagong/chittagong_driver.rb +47 -0
  38. data/test/{chittagong_driver_test.rb → chittagong/chittagong_driver_test.rb} +66 -58
  39. data/test/{chittagong_evaluator.rb → chittagong/chittagong_evaluator.rb} +28 -13
  40. data/test/{chittagong_evaluator_test.rb → chittagong/chittagong_evaluator_test.rb} +6 -10
  41. data/test/chittagong/chittagong_grammar.rb +110 -0
  42. data/test/{chittagong_parser_test.rb → chittagong/chittagong_parser_test.rb} +5 -7
  43. data/test/{chittagong_test.rb → chittagong/chittagong_test.rb} +27 -36
  44. data/test/{chittagong_tokenizer.rb → chittagong/chittagong_tokenizer.rb} +17 -17
  45. data/test/{chittagong_tokenizer_test.rb → chittagong/chittagong_tokenizer_test.rb} +2 -3
  46. data/test/compiled_parser_test.rb +9 -42
  47. data/test/dhaka_test_helper.rb +17 -0
  48. data/test/evaluator_test.rb +18 -3
  49. data/test/grammar_test.rb +10 -15
  50. data/test/lalr_but_not_slr_grammar.rb +10 -8
  51. data/test/malformed_grammar.rb +2 -4
  52. data/test/malformed_grammar_test.rb +2 -3
  53. data/test/nullable_grammar.rb +11 -8
  54. data/test/parse_result_test.rb +44 -0
  55. data/test/parser_state_test.rb +36 -0
  56. data/test/parser_test.rb +53 -103
  57. data/test/precedence_grammar.rb +6 -6
  58. data/test/precedence_grammar_test.rb +2 -3
  59. data/test/rr_conflict_grammar.rb +5 -7
  60. data/test/simple_grammar.rb +6 -8
  61. data/test/sr_conflict_grammar.rb +6 -6
  62. metadata +30 -26
  63. data/test/arithmetic_grammar.rb +0 -35
  64. data/test/arithmetic_precedence_grammar.rb +0 -24
  65. data/test/arithmetic_precedence_parser_test.rb +0 -33
  66. data/test/bracket_grammar.rb +0 -25
  67. data/test/chittagong_grammar.rb +0 -104
  68. data/test/incomplete_arithmetic_evaluator.rb +0 -60
@@ -1,21 +1,20 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
- require 'chittagong_grammar'
3
-
1
+ require File.dirname(__FILE__) + '/chittagong_grammar'
4
2
 
5
3
  class ChittagongTokenizer < Dhaka::Tokenizer
6
4
 
7
- KEYWORDS = ['print', 'if', 'else', 'end', 'while', 'def', 'return']
5
+ KEYWORDS = %w| print if else end while def return |
8
6
 
9
- digits = ('0'..'9').to_a
10
- letters = ('a'..'z').to_a
11
- parenths = ['(', ')']
12
- operators = ['-', '+', '/', '*', '^', '!', '>', '<']
13
- equal_sign = ['=']
14
- whitespace = [' ']
15
- arg_separator = [',']
16
- newline = ["\n"]
7
+ digits = ('0'..'9').to_a
8
+ decimal_point = %w| . |
9
+ letters = ('a'..'z').to_a
10
+ parenths = %w| ( ) |
11
+ operators = %w| - + / * ^ ! > < |
12
+ equal_sign = %w| = |
13
+ whitespace = [' ']
14
+ arg_separator = %w| , |
15
+ newline = ["\n"]
17
16
 
18
- all_characters = digits + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
17
+ all_characters = digits + decimal_point + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
19
18
 
20
19
  for_state Dhaka::TOKENIZER_IDLE_STATE do
21
20
  for_characters(all_characters - (digits + letters + newline + whitespace + equal_sign)) do
@@ -49,7 +48,7 @@ class ChittagongTokenizer < Dhaka::Tokenizer
49
48
  switch_to Dhaka::TOKENIZER_IDLE_STATE
50
49
  end
51
50
  for_character equal_sign do
52
- curr_token.symbol_name += '='
51
+ curr_token.symbol_name << '='
53
52
  advance
54
53
  switch_to Dhaka::TOKENIZER_IDLE_STATE
55
54
  end
@@ -60,18 +59,18 @@ class ChittagongTokenizer < Dhaka::Tokenizer
60
59
  switch_to Dhaka::TOKENIZER_IDLE_STATE
61
60
  end
62
61
  for_characters digits do
63
- curr_token.value += curr_char
62
+ curr_token.value << curr_char
64
63
  advance
65
64
  end
66
65
  end
67
-
66
+ # lipi:keywords
68
67
  for_state :get_word_literal do
69
68
  for_characters all_characters - letters do
70
69
  curr_token.symbol_name = word_literal_symbol(curr_token.value)
71
70
  switch_to Dhaka::TOKENIZER_IDLE_STATE
72
71
  end
73
72
  for_characters letters do
74
- curr_token.value += curr_char
73
+ curr_token.value << curr_char
75
74
  advance
76
75
  curr_token.symbol_name = word_literal_symbol(curr_token.value) unless curr_char
77
76
  end
@@ -84,5 +83,6 @@ class ChittagongTokenizer < Dhaka::Tokenizer
84
83
  'word_literal'
85
84
  end
86
85
  end
86
+ # lipi:keywords
87
87
 
88
88
  end
@@ -1,6 +1,5 @@
1
- require "test/unit"
2
-
3
- require "chittagong_tokenizer"
1
+ require File.dirname(__FILE__) + '/../dhaka_test_helper'
2
+ require File.dirname(__FILE__) + "/chittagong_tokenizer"
4
3
 
5
4
  class TestChittagongTokenizer < Test::Unit::TestCase
6
5
  def test_tokenizes_a_program
@@ -1,16 +1,10 @@
1
- require "test/unit"
2
- require "simple_grammar"
3
- require 'arithmetic_test_methods'
4
- require 'arithmetic_grammar'
5
-
6
- eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as('Foo'))
7
- eval(Dhaka::Parser.new(ArithmeticGrammar).compile_to_ruby_source_as('CompiledArithmeticParser'))
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + "/simple_grammar"
3
+ eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as(:SimpleParser))
8
4
 
9
5
  class TestCompiledParser < Test::Unit::TestCase
10
- include ArithmeticTestMethods
11
-
12
6
  def test_compiled_parser_generates_parse_tree_for_simple_grammar
13
- parse_tree = Foo.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME])).parse_tree
7
+ parse_tree = SimpleParser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME]))
14
8
  assert_equal \
15
9
  ["literal",
16
10
  "term",
@@ -28,39 +22,13 @@ class TestCompiledParser < Test::Unit::TestCase
28
22
  "start"], parse_tree.linearize.collect {|node| node.production.name}
29
23
  end
30
24
 
31
- def test_compiled_parser_generates_parse_tree_for_arithmetic_grammar
32
- parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')',Dhaka::END_SYMBOL_NAME]
33
- assert_equal \
34
- ["getting_literals",
35
- "factor",
36
- "term",
37
- "getting_literals",
38
- "factor",
39
- "getting_literals",
40
- "division",
41
- "term",
42
- "getting_literals",
43
- "factor",
44
- "subtraction",
45
- "unpacking_parenthetized_expression",
46
- "factor",
47
- "getting_literals",
48
- "division",
49
- "subtraction",
50
- "unpacking_parenthetized_expression",
51
- "factor",
52
- "term",
53
- "expression"], parse(build_tokens(parser_input)).linearize.collect {|node| node.production.name}
54
- end
55
-
56
- def test_parse_result_has_nil_parse_tree_if_empty_token_array
57
- parse_result = CompiledArithmeticParser.parse([])
58
- assert !CompiledArithmeticParser.parse([]).has_error?
59
- assert_nil parse_result.parse_tree
25
+ def test_parse_result_has_error_result_if_only_end_token_passed
26
+ parse_result = SimpleParser.parse(build_tokens([Dhaka::END_SYMBOL_NAME]))
27
+ assert parse_result.has_error?
60
28
  end
61
29
 
62
30
  def test_parser_returns_error_result_with_index_of_bad_token_if_parse_error
63
- parse_result = CompiledArithmeticParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
31
+ parse_result = SimpleParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
64
32
  assert parse_result.has_error?
65
33
  assert_equal '-', parse_result.unexpected_token.symbol_name
66
34
  end
@@ -68,5 +36,4 @@ class TestCompiledParser < Test::Unit::TestCase
68
36
  def build_tokens(token_symbol_names)
69
37
  token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
70
38
  end
71
-
72
- end
39
+ end
@@ -0,0 +1,17 @@
1
+ unless defined? Dhaka
2
+ require File.dirname(__FILE__) + '/../lib/dhaka'
3
+ end
4
+
5
+ require File.dirname(__FILE__) + '/fake_logger'
6
+
7
+ require 'test/unit'
8
+
9
+ begin
10
+ require 'mocha'
11
+ rescue LoadError
12
+ puts "
13
+ The tests depend on Mocha. Please install it if you wish to run them:
14
+ sudo gem install mocha
15
+ This gem is not required for using the library."
16
+ exit
17
+ end
@@ -1,8 +1,23 @@
1
- require "test/unit"
2
- require File.dirname(__FILE__)+'/../lib/dhaka'
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
3
3
 
4
4
  class TestEvaluator < Test::Unit::TestCase
5
5
  def test_throws_exception_if_evaluation_rules_not_completely_defined
6
- assert_raise(Dhaka::EvaluatorDefinitionError) {require 'incomplete_arithmetic_evaluator'}
6
+ assert_raise(Dhaka::EvaluatorDefinitionError) do
7
+ eval(
8
+ "class IncompleteSimpleEvaluator < Dhaka::Evaluator
9
+ self.grammar = SimpleGrammar
10
+ define_evaluation_rules do
11
+ for_start do
12
+ something
13
+ end
14
+
15
+ for_literal do
16
+ something
17
+ end
18
+ end
19
+ end")
20
+ end
21
+
7
22
  end
8
23
  end
@@ -1,35 +1,32 @@
1
- #!/usr/bin/env ruby
2
- require 'test/unit'
3
- require 'simple_grammar'
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
4
3
 
5
4
  class SimpleGrammarTest < Test::Unit::TestCase
6
-
7
5
  def setup
8
6
  @grammar = SimpleGrammar
9
7
  end
10
8
 
11
9
  def test_loads_symbol_and_classifies_them
12
10
  expected_non_terminals = Set.new(['E', 'S', 'T', Dhaka::START_SYMBOL_NAME])
13
- expected_terminals = Set.new(['-', 'n', '(', ')', '#', Dhaka::END_SYMBOL_NAME])
11
+ expected_terminals = Set.new(['-', 'n', '(', ')', '#', Dhaka::END_SYMBOL_NAME])
14
12
  assert_equal(expected_non_terminals, Set.new(@grammar.non_terminal_symbols.collect {|symbol| symbol.name}))
15
13
  assert_equal(expected_terminals, Set.new(@grammar.terminal_symbols.collect {|symbol| symbol.name}))
16
14
  end
17
15
 
18
16
  def test_creates_productions
19
17
  productions_for_E = @grammar.productions_for_symbol(@grammar.symbol_for_name('E'))
20
- assert(productions_for_E.size==2)
18
+ assert(productions_for_E.size == 2)
21
19
  expected_productions_for_E = Set.new(['subtraction E ::= E - T', 'term E ::= T'])
22
20
  assert_equal(expected_productions_for_E, Set.new(productions_for_E.collect {|production| production.to_s}))
23
21
  productions_for_start = @grammar.productions_for_symbol(@grammar.start_symbol)
24
- assert(productions_for_start.size==1)
22
+ assert(productions_for_start.size == 1)
25
23
  expected_productions_for_start = Set.new(['start _Start_ ::= S #'])
26
24
  assert_equal(expected_productions_for_start, Set.new(productions_for_start.collect {|production| production.to_s}))
27
-
28
25
  end
29
26
 
30
27
  def test_symbols_in_productions_use_the_flyweight_pattern
31
28
  assert_same(@grammar.production_named('subtraction').symbol, @grammar.production_named('term').symbol)
32
- assert_same(@grammar.production_named('expression').expansion[0], @grammar.production_named('subtraction').expansion[0])
29
+ assert_same(@grammar.production_named('expression').expansion.first, @grammar.production_named('subtraction').expansion.first)
33
30
  end
34
31
 
35
32
  def test_first_with_non_terminal
@@ -43,11 +40,11 @@ class SimpleGrammarTest < Test::Unit::TestCase
43
40
  end
44
41
 
45
42
  def test_computes_closures_and_channels_given_a_kernel
46
- start_production = @grammar.production_named('start')
47
- start_item = Dhaka::Item.new(start_production, 0)
48
- kernel = Set.new([start_item])
43
+ start_production = @grammar.production_named('start')
44
+ start_item = Dhaka::Item.new(start_production, 0)
45
+ kernel = Set.new([start_item])
49
46
  channels, closure = @grammar.closure(kernel)
50
- expected_items = Set.new(['_Start_ ::= -> S # []',
47
+ expected_items = Set.new(['_Start_ ::= -> S # []',
51
48
  'S ::= -> E []',
52
49
  'E ::= -> E - T []',
53
50
  'E ::= -> T []',
@@ -65,6 +62,4 @@ class SimpleGrammarTest < Test::Unit::TestCase
65
62
  assert_equal(expected_items, Set.new(closure.values.collect{|item| item.to_s}))
66
63
  assert_equal(expected_channels, Set.new(channels.collect{|item| item.to_s}))
67
64
  end
68
-
69
-
70
65
  end
@@ -1,17 +1,19 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
-
3
1
  class LALRButNotSLRGrammar < Dhaka::Grammar
2
+
4
3
  for_symbol(Dhaka::START_SYMBOL_NAME) do
5
- start ['E']
4
+ start %w| E |
6
5
  end
6
+
7
7
  for_symbol('E') do
8
- E_Aa ['A', 'a']
9
- E_bAc ['b', 'A', 'c']
10
- E_dc ['d', 'c']
11
- E_bda ['b', 'd', 'a']
8
+ E_Aa %w| A a |
9
+ E_bAc %w| b A c |
10
+ E_dc %w| d c |
11
+ E_bda %w| b d a |
12
12
  end
13
+
13
14
  for_symbol('A') do
14
- A_d ['d']
15
+ A_d %w| d |
15
16
  end
17
+
16
18
  end
17
19
 
@@ -1,9 +1,7 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
-
3
1
  class MalformedGrammar < Dhaka::Grammar
4
2
 
5
- for_symbol('goo') do
6
- foo ['boo']
3
+ for_symbol('foo') do
4
+ bar %w| baz |
7
5
  end
8
6
 
9
7
  end
@@ -1,6 +1,5 @@
1
- require "test/unit"
2
-
3
- require "malformed_grammar"
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + "/malformed_grammar"
4
3
 
5
4
  class TestMalformedGrammar < Test::Unit::TestCase
6
5
  def test_must_have_a_start_symbol_in_order_to_generate_a_parser
@@ -1,18 +1,21 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
-
3
1
  class NullableGrammar < Dhaka::Grammar
2
+
4
3
  for_symbol(Dhaka::START_SYMBOL_NAME) do
5
- tuple ['Tuple']
4
+ tuple %w| Tuple |
6
5
  end
6
+
7
7
  for_symbol('Tuple') do
8
- element_list ['(', 'Elements', ')']
8
+ element_list %w| ( Elements ) |
9
9
  end
10
+
10
11
  for_symbol('Elements') do
11
- empty_element_list []
12
- concatenate_element_lists ['Character', 'Elements']
12
+ empty_element_list %w| |
13
+ concatenate_element_lists %w| Character Elements |
13
14
  end
15
+
14
16
  for_symbol('Character') do
15
- literal_a ['a']
16
- literal_b ['b']
17
+ literal_a %w| a |
18
+ literal_b %w| b |
17
19
  end
20
+
18
21
  end
@@ -0,0 +1,44 @@
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
3
+
4
+ class TestParseSuccessResult < Test::Unit::TestCase
5
+ include Dhaka
6
+
7
+ def composite_node(production, child_nodes, dot_name)
8
+ node = ParseTreeCompositeNode.new(SimpleGrammar.production_named(production))
9
+ node.child_nodes.concat child_nodes
10
+ node.stubs(:dot_name).returns(dot_name)
11
+ node
12
+ end
13
+
14
+ def leaf_node(token, value, dot_name)
15
+ node = ParseTreeLeafNode.new(Token.new(token, value, nil))
16
+ node.stubs(:dot_name).returns(dot_name)
17
+ node
18
+ end
19
+
20
+ def test_parse_tree_can_be_exported_to_dot_format
21
+ first_term = composite_node('literal', [leaf_node('n', 1, "literal_1")], "first_term")
22
+ second_term = composite_node('literal', [leaf_node('n', 2, "literal_2")], "second_term")
23
+ addition_term = leaf_node('-', nil, "subtraction_operator")
24
+ tree = composite_node('subtraction', [first_term, addition_term, second_term], "expression")
25
+ result = ParseSuccessResult.new(tree)
26
+ assert_equal(
27
+ %(digraph x {
28
+ node [fontsize="10" shape=box size="5"]
29
+ expression [label="subtraction E ::= E - T"]
30
+ expression -> first_term
31
+ first_term [label="literal T ::= n"]
32
+ first_term -> literal_1
33
+ literal_1 [label="n : 1"]
34
+ expression -> subtraction_operator
35
+ subtraction_operator [label="-"]
36
+ expression -> second_term
37
+ second_term [label="literal T ::= n"]
38
+ second_term -> literal_2
39
+ literal_2 [label="n : 2"]
40
+ }),
41
+ result.to_dot)
42
+ end
43
+
44
+ end
@@ -0,0 +1,36 @@
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+
3
+ class TestParserState < Test::Unit::TestCase
4
+ include Dhaka
5
+ def test_unique_identifier
6
+ states = (0...5).collect {|i| ParserState.new(nil, {})}
7
+ ids = Set.new
8
+ states.each do |state|
9
+ assert(/^State(\d+)$/ =~ state.unique_name)
10
+ ids << $1.to_i
11
+ end
12
+ assert_equal(5, ids.size)
13
+ end
14
+
15
+ def test_to_s_method
16
+ options = {:some_option => true}
17
+ item1 = mock()
18
+ item1.stubs(:to_s).with(options).returns("i'm item 1")
19
+ item2 = mock()
20
+ item2.stubs(:to_s).with(options).returns("i'm item 2")
21
+ state = ParserState.new(nil, {item1 => item1, item2 => item2})
22
+ assert_equal(["i'm item 1", "i'm item 2"], state.to_s(options).split("\n").sort)
23
+ end
24
+
25
+ def test_to_dot_method
26
+ options = {:some_option => true}
27
+ item1 = mock()
28
+ item1.stubs(:to_s).with(options).returns("i'm item 1")
29
+ item2 = mock()
30
+ item2.stubs(:to_s).with(options).returns("i'm item 2")
31
+ state = ParserState.new(nil, {item1 => item1, item2 => item2})
32
+ state.stubs(:unique_name).returns("State1")
33
+ assert(state.to_dot(options) =~ /^State1 \[label="(.+)\\n(.+)"\]/)
34
+ assert_equal(["i'm item 1", "i'm item 2"], [$1, $2].sort)
35
+ end
36
+ end
@@ -1,16 +1,12 @@
1
- #!/usr/bin/env ruby
2
- require 'test/unit'
3
- require 'simple_grammar'
4
- require 'arithmetic_grammar'
5
- require 'nullable_grammar'
6
- require 'lalr_but_not_slr_grammar'
7
- require 'another_lalr_but_not_slr_grammar'
8
- require 'rr_conflict_grammar'
9
- require 'sr_conflict_grammar'
10
- require 'fake_logger'
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
3
+ require File.dirname(__FILE__) + '/nullable_grammar'
4
+ require File.dirname(__FILE__) + '/lalr_but_not_slr_grammar'
5
+ require File.dirname(__FILE__) + '/another_lalr_but_not_slr_grammar'
6
+ require File.dirname(__FILE__) + '/rr_conflict_grammar'
7
+ require File.dirname(__FILE__) + '/sr_conflict_grammar'
11
8
 
12
9
  class ParserTest < Test::Unit::TestCase
13
-
14
10
  def build_tokens(token_symbol_names)
15
11
  token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
16
12
  end
@@ -20,37 +16,45 @@ class ParserTest < Test::Unit::TestCase
20
16
  @parser = Dhaka::Parser.new(@grammar)
21
17
  end
22
18
 
19
+ def contains(given_member, set)
20
+ set.inject(false) {|result, member| result ||= (Set.new(member) == Set.new(given_member))}
21
+ end
22
+
23
+ def assert_collection_equal(expected, actual)
24
+ assert_equal(expected.size, actual.size)
25
+ actual.each do |actual_member|
26
+ assert(contains(actual_member, expected))
27
+ end
28
+ end
29
+
23
30
  def test_parser_generates_states_with_correct_items
24
- expected_states = {}
25
- expected_states[1] = Set.new(['_Start_ ::= -> S # [_End_]',
31
+ expected_states = [
32
+ ['_Start_ ::= -> S # [_End_]',
26
33
  'S ::= -> E [#]',
27
34
  'E ::= -> E - T [#-]',
28
35
  'E ::= -> T [#-]',
29
36
  'T ::= -> n [#-]',
30
- 'T ::= -> ( E ) [#-]'])
31
- expected_states[2] = Set.new(['E ::= T -> [#)-]'])
32
- expected_states[3] = Set.new(['T ::= n -> [#)-]'])
33
- expected_states[4] = Set.new(['S ::= E -> [#]',
34
- 'E ::= E -> - T [#-]'])
35
- expected_states[5] = Set.new(['_Start_ ::= S -> # [_End_]'])
36
- expected_states[6] = Set.new(['T ::= ( -> E ) [#)-]',
37
+ 'T ::= -> ( E ) [#-]'],
38
+ ['E ::= T -> [#)-]'],
39
+ ['T ::= n -> [#)-]'],
40
+ ['S ::= E -> [#]',
41
+ 'E ::= E -> - T [#-]'],
42
+ ['_Start_ ::= S -> # [_End_]'],
43
+ ['T ::= ( -> E ) [#)-]',
37
44
  'E ::= -> E - T [)-]',
38
45
  'E ::= -> T [)-]',
39
46
  'T ::= -> n [)-]',
40
- 'T ::= -> ( E ) [)-]'])
41
- expected_states[7] = Set.new(['E ::= E - -> T [#)-]',
47
+ 'T ::= -> ( E ) [)-]'],
48
+ ['E ::= E - -> T [#)-]',
42
49
  'T ::= -> n [#)-]',
43
- 'T ::= -> ( E ) [#)-]'])
44
- expected_states[8] = Set.new(['E ::= E - T -> [#)-]'])
45
- expected_states[9] = Set.new(['T ::= ( E -> ) [#)-]',
46
- 'E ::= E -> - T [)-]'])
47
- expected_states[10] = Set.new(['T ::= ( E ) -> [#)-]'])
48
- expected_states[11] = Set.new(['_Start_ ::= S # -> [_End_]'])
49
- actual_states = Set.new(@parser.send('states').collect {|state| Set.new(state.items.values.collect {|item| item.to_s})})
50
- #write_parser(@parser)
51
- expected_states.values.each do |state|
52
- assert set_finder(state, actual_states), "expected #{state.to_a}"
53
- end
50
+ 'T ::= -> ( E ) [#)-]'],
51
+ ['E ::= E - T -> [#)-]'],
52
+ ['T ::= ( E -> ) [#)-]',
53
+ 'E ::= E -> - T [)-]'],
54
+ ['T ::= ( E ) -> [#)-]'],
55
+ ['_Start_ ::= S # -> [_End_]']]
56
+ actual_states = @parser.send('states').collect {|state| state.items.values.collect {|item| item.to_s}}
57
+ assert_collection_equal(expected_states, actual_states)
54
58
  end
55
59
 
56
60
  def test_parser_can_be_exported_to_dot_format
@@ -58,7 +62,7 @@ class ParserTest < Test::Unit::TestCase
58
62
  end
59
63
 
60
64
  def test_parser_generates_parse_tree_given_a_stream_of_symbols
61
- parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME])).parse_tree
65
+ parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME]))
62
66
  assert_equal \
63
67
  ["literal",
64
68
  "term",
@@ -76,68 +80,13 @@ class ParserTest < Test::Unit::TestCase
76
80
  "start"], parse_tree.linearize.collect {|node| node.production.name}
77
81
  end
78
82
 
79
- def test_parse_trees_can_be_exported_to_dot_format
80
- parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME])).parse_tree
81
- parse_tree.to_dot
82
- end
83
-
84
83
  def get_linearized_parse_result(input, parser)
85
84
  parser.parse(build_tokens(input)).parse_tree.linearize.collect {|node| node.production.name}
86
85
  end
87
86
 
88
- def test_with_a_different_grammar_with_division
89
- grammar = ArithmeticGrammar
90
- parser = Dhaka::Parser.new(grammar)
91
- parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')', Dhaka::END_SYMBOL_NAME]
92
- assert_equal \
93
- ["getting_literals",
94
- "factor",
95
- "term",
96
- "getting_literals",
97
- "factor",
98
- "getting_literals",
99
- "division",
100
- "term",
101
- "getting_literals",
102
- "factor",
103
- "subtraction",
104
- "unpacking_parenthetized_expression",
105
- "factor",
106
- "getting_literals",
107
- "division",
108
- "subtraction",
109
- "unpacking_parenthetized_expression",
110
- "factor",
111
- "term",
112
- "expression"], get_linearized_parse_result(parser_input, parser)
113
-
114
- parser_input = ['h','(','(','n',')','-','n',',','n',')', Dhaka::END_SYMBOL_NAME]
115
- assert_equal \
116
- ["max_function",
117
- "getting_literals",
118
- "factor",
119
- "term",
120
- "unpacking_parenthetized_expression",
121
- "factor",
122
- "term",
123
- "getting_literals",
124
- "factor",
125
- "subtraction",
126
- "getting_literals",
127
- "factor",
128
- "term",
129
- "single_args",
130
- "concatenating_args",
131
- "evaluating_function",
132
- "function",
133
- "factor",
134
- "term",
135
- "expression"], get_linearized_parse_result(parser_input, parser)
136
- end
137
-
138
87
  def test_with_a_grammar_with_nullables_after_terminals
139
- grammar = NullableGrammar
140
- parser = Dhaka::Parser.new(grammar)
88
+ grammar = NullableGrammar
89
+ parser = Dhaka::Parser.new(grammar)
141
90
  parser_input = ['(','a',')', Dhaka::END_SYMBOL_NAME]
142
91
  assert_equal \
143
92
  ["literal_a",
@@ -148,30 +97,31 @@ class ParserTest < Test::Unit::TestCase
148
97
  end
149
98
 
150
99
  def test_with_a_grammar_that_is_not_SLR
151
- grammar = LALRButNotSLRGrammar
152
- parser = Dhaka::Parser.new(grammar)
100
+ grammar = LALRButNotSLRGrammar
101
+ parser = Dhaka::Parser.new(grammar)
153
102
  parser_input = ['b','d','c', Dhaka::END_SYMBOL_NAME]
154
103
  assert_equal(["A_d", "E_bAc", "start"], get_linearized_parse_result(parser_input, parser))
155
104
  end
156
105
 
157
106
  def test_with_another_grammar_that_is_not_SLR
158
- grammar = AnotherLALRButNotSLRGrammar
159
- parser = Dhaka::Parser.new(grammar)
107
+ grammar = AnotherLALRButNotSLRGrammar
108
+ parser = Dhaka::Parser.new(grammar)
160
109
  parser_input = ['*', 'id', '=', 'id', Dhaka::END_SYMBOL_NAME]
161
110
  assert_equal(["identifier", "l_value", "contents", "identifier", "l_value", "assignment"], get_linearized_parse_result(parser_input, parser))
162
111
  end
163
112
 
164
- def test_with_a_grammar_that_should_generate_an_RR_conflict
165
- grammar = RRConflictGrammar
113
+ def test_debug_output_with_a_grammar_that_should_generate_an_RR_conflict
166
114
  fake_logger = FakeLogger.new
167
- Dhaka::Parser.new(grammar, fake_logger)
168
- assert_equal(1, fake_logger.errors.size)
115
+ Dhaka::ParserState.any_instance.stubs(:unique_name).returns("StateXXX")
116
+ parser = Dhaka::Parser.new(RRConflictGrammar, fake_logger)
117
+ num_states = parser.send(:states).size
118
+ assert_equal(['Created StateXXX.'] * num_states, fake_logger.debugs[0...num_states])
119
+ assert_equal(1, fake_logger.errors.size)
120
+ assert(fake_logger.errors.first.match(/^Parser Conflict at State:\n(.+)\n(.+)\nExisting: (.+)\nNew: (.+)\nLookahead: (.+)$/))
121
+ assert_equal(["A ::= x y ->", "B ::= x y ->", "Reduce with xy A ::= x y", "Reduce with xy_again B ::= x y", "c"],
122
+ $~[1..5].sort)
169
123
  end
170
124
 
171
- def set_finder(set1, set2)
172
- set2.inject(false) {|result, member| result ||= member == set1}
173
- end
174
-
175
125
  def write_parser(parser)
176
126
  File.open('parser.dot', 'w') do |file|
177
127
  file << parser.to_dot