dhaka 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. data/lib/dhaka.rb +24 -22
  2. data/lib/evaluator/evaluator.rb +42 -44
  3. data/lib/grammar/closure_hash.rb +4 -3
  4. data/lib/grammar/grammar.rb +113 -110
  5. data/lib/grammar/grammar_symbol.rb +6 -3
  6. data/lib/grammar/precedence.rb +3 -2
  7. data/lib/grammar/production.rb +5 -6
  8. data/lib/parser/action.rb +16 -11
  9. data/lib/parser/channel.rb +22 -16
  10. data/lib/parser/compiled_parser.rb +28 -22
  11. data/lib/parser/conflict.rb +54 -0
  12. data/lib/parser/item.rb +19 -19
  13. data/lib/parser/parse_result.rb +16 -1
  14. data/lib/parser/parse_tree.rb +15 -9
  15. data/lib/parser/parser.rb +51 -80
  16. data/lib/parser/parser_run.rb +6 -6
  17. data/lib/parser/parser_state.rb +16 -18
  18. data/lib/parser/token.rb +6 -4
  19. data/lib/tokenizer/tokenizer.rb +34 -31
  20. data/test/all_tests.rb +4 -18
  21. data/test/another_lalr_but_not_slr_grammar.rb +9 -5
  22. data/test/{arithmetic_evaluator.rb → arithmetic/arithmetic_evaluator.rb} +1 -2
  23. data/test/{arithmetic_evaluator_test.rb → arithmetic/arithmetic_evaluator_test.rb} +9 -20
  24. data/test/arithmetic/arithmetic_grammar.rb +41 -0
  25. data/test/{arithmetic_grammar_test.rb → arithmetic/arithmetic_grammar_test.rb} +2 -4
  26. data/test/{arithmetic_test_methods.rb → arithmetic/arithmetic_test_methods.rb} +1 -3
  27. data/test/{arithmetic_tokenizer.rb → arithmetic/arithmetic_tokenizer.rb} +8 -10
  28. data/test/{arithmetic_tokenizer_test.rb → arithmetic/arithmetic_tokenizer_test.rb} +4 -2
  29. data/test/{arithmetic_precedence_evaluator.rb → arithmetic_precedence/arithmetic_precedence_evaluator.rb} +1 -2
  30. data/test/arithmetic_precedence/arithmetic_precedence_grammar.rb +24 -0
  31. data/test/{arithmetic_precedence_grammar_test.rb → arithmetic_precedence/arithmetic_precedence_grammar_test.rb} +2 -3
  32. data/test/arithmetic_precedence/arithmetic_precedence_parser_test.rb +31 -0
  33. data/test/{arithmetic_precedence_tokenizer.rb → arithmetic_precedence/arithmetic_precedence_tokenizer.rb} +8 -10
  34. data/test/brackets/bracket_grammar.rb +23 -0
  35. data/test/{bracket_tokenizer.rb → brackets/bracket_tokenizer.rb} +2 -4
  36. data/test/{brackets_test.rb → brackets/brackets_test.rb} +3 -4
  37. data/test/chittagong/chittagong_driver.rb +47 -0
  38. data/test/{chittagong_driver_test.rb → chittagong/chittagong_driver_test.rb} +66 -58
  39. data/test/{chittagong_evaluator.rb → chittagong/chittagong_evaluator.rb} +28 -13
  40. data/test/{chittagong_evaluator_test.rb → chittagong/chittagong_evaluator_test.rb} +6 -10
  41. data/test/chittagong/chittagong_grammar.rb +110 -0
  42. data/test/{chittagong_parser_test.rb → chittagong/chittagong_parser_test.rb} +5 -7
  43. data/test/{chittagong_test.rb → chittagong/chittagong_test.rb} +27 -36
  44. data/test/{chittagong_tokenizer.rb → chittagong/chittagong_tokenizer.rb} +17 -17
  45. data/test/{chittagong_tokenizer_test.rb → chittagong/chittagong_tokenizer_test.rb} +2 -3
  46. data/test/compiled_parser_test.rb +9 -42
  47. data/test/dhaka_test_helper.rb +17 -0
  48. data/test/evaluator_test.rb +18 -3
  49. data/test/grammar_test.rb +10 -15
  50. data/test/lalr_but_not_slr_grammar.rb +10 -8
  51. data/test/malformed_grammar.rb +2 -4
  52. data/test/malformed_grammar_test.rb +2 -3
  53. data/test/nullable_grammar.rb +11 -8
  54. data/test/parse_result_test.rb +44 -0
  55. data/test/parser_state_test.rb +36 -0
  56. data/test/parser_test.rb +53 -103
  57. data/test/precedence_grammar.rb +6 -6
  58. data/test/precedence_grammar_test.rb +2 -3
  59. data/test/rr_conflict_grammar.rb +5 -7
  60. data/test/simple_grammar.rb +6 -8
  61. data/test/sr_conflict_grammar.rb +6 -6
  62. metadata +30 -26
  63. data/test/arithmetic_grammar.rb +0 -35
  64. data/test/arithmetic_precedence_grammar.rb +0 -24
  65. data/test/arithmetic_precedence_parser_test.rb +0 -33
  66. data/test/bracket_grammar.rb +0 -25
  67. data/test/chittagong_grammar.rb +0 -104
  68. data/test/incomplete_arithmetic_evaluator.rb +0 -60
@@ -1,21 +1,20 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
- require 'chittagong_grammar'
3
-
1
+ require File.dirname(__FILE__) + '/chittagong_grammar'
4
2
 
5
3
  class ChittagongTokenizer < Dhaka::Tokenizer
6
4
 
7
- KEYWORDS = ['print', 'if', 'else', 'end', 'while', 'def', 'return']
5
+ KEYWORDS = %w| print if else end while def return |
8
6
 
9
- digits = ('0'..'9').to_a
10
- letters = ('a'..'z').to_a
11
- parenths = ['(', ')']
12
- operators = ['-', '+', '/', '*', '^', '!', '>', '<']
13
- equal_sign = ['=']
14
- whitespace = [' ']
15
- arg_separator = [',']
16
- newline = ["\n"]
7
+ digits = ('0'..'9').to_a
8
+ decimal_point = %w| . |
9
+ letters = ('a'..'z').to_a
10
+ parenths = %w| ( ) |
11
+ operators = %w| - + / * ^ ! > < |
12
+ equal_sign = %w| = |
13
+ whitespace = [' ']
14
+ arg_separator = %w| , |
15
+ newline = ["\n"]
17
16
 
18
- all_characters = digits + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
17
+ all_characters = digits + decimal_point + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
19
18
 
20
19
  for_state Dhaka::TOKENIZER_IDLE_STATE do
21
20
  for_characters(all_characters - (digits + letters + newline + whitespace + equal_sign)) do
@@ -49,7 +48,7 @@ class ChittagongTokenizer < Dhaka::Tokenizer
49
48
  switch_to Dhaka::TOKENIZER_IDLE_STATE
50
49
  end
51
50
  for_character equal_sign do
52
- curr_token.symbol_name += '='
51
+ curr_token.symbol_name << '='
53
52
  advance
54
53
  switch_to Dhaka::TOKENIZER_IDLE_STATE
55
54
  end
@@ -60,18 +59,18 @@ class ChittagongTokenizer < Dhaka::Tokenizer
60
59
  switch_to Dhaka::TOKENIZER_IDLE_STATE
61
60
  end
62
61
  for_characters digits do
63
- curr_token.value += curr_char
62
+ curr_token.value << curr_char
64
63
  advance
65
64
  end
66
65
  end
67
-
66
+ # lipi:keywords
68
67
  for_state :get_word_literal do
69
68
  for_characters all_characters - letters do
70
69
  curr_token.symbol_name = word_literal_symbol(curr_token.value)
71
70
  switch_to Dhaka::TOKENIZER_IDLE_STATE
72
71
  end
73
72
  for_characters letters do
74
- curr_token.value += curr_char
73
+ curr_token.value << curr_char
75
74
  advance
76
75
  curr_token.symbol_name = word_literal_symbol(curr_token.value) unless curr_char
77
76
  end
@@ -84,5 +83,6 @@ class ChittagongTokenizer < Dhaka::Tokenizer
84
83
  'word_literal'
85
84
  end
86
85
  end
86
+ # lipi:keywords
87
87
 
88
88
  end
@@ -1,6 +1,5 @@
1
- require "test/unit"
2
-
3
- require "chittagong_tokenizer"
1
+ require File.dirname(__FILE__) + '/../dhaka_test_helper'
2
+ require File.dirname(__FILE__) + "/chittagong_tokenizer"
4
3
 
5
4
  class TestChittagongTokenizer < Test::Unit::TestCase
6
5
  def test_tokenizes_a_program
@@ -1,16 +1,10 @@
1
- require "test/unit"
2
- require "simple_grammar"
3
- require 'arithmetic_test_methods'
4
- require 'arithmetic_grammar'
5
-
6
- eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as('Foo'))
7
- eval(Dhaka::Parser.new(ArithmeticGrammar).compile_to_ruby_source_as('CompiledArithmeticParser'))
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + "/simple_grammar"
3
+ eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as(:SimpleParser))
8
4
 
9
5
  class TestCompiledParser < Test::Unit::TestCase
10
- include ArithmeticTestMethods
11
-
12
6
  def test_compiled_parser_generates_parse_tree_for_simple_grammar
13
- parse_tree = Foo.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME])).parse_tree
7
+ parse_tree = SimpleParser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME]))
14
8
  assert_equal \
15
9
  ["literal",
16
10
  "term",
@@ -28,39 +22,13 @@ class TestCompiledParser < Test::Unit::TestCase
28
22
  "start"], parse_tree.linearize.collect {|node| node.production.name}
29
23
  end
30
24
 
31
- def test_compiled_parser_generates_parse_tree_for_arithmetic_grammar
32
- parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')',Dhaka::END_SYMBOL_NAME]
33
- assert_equal \
34
- ["getting_literals",
35
- "factor",
36
- "term",
37
- "getting_literals",
38
- "factor",
39
- "getting_literals",
40
- "division",
41
- "term",
42
- "getting_literals",
43
- "factor",
44
- "subtraction",
45
- "unpacking_parenthetized_expression",
46
- "factor",
47
- "getting_literals",
48
- "division",
49
- "subtraction",
50
- "unpacking_parenthetized_expression",
51
- "factor",
52
- "term",
53
- "expression"], parse(build_tokens(parser_input)).linearize.collect {|node| node.production.name}
54
- end
55
-
56
- def test_parse_result_has_nil_parse_tree_if_empty_token_array
57
- parse_result = CompiledArithmeticParser.parse([])
58
- assert !CompiledArithmeticParser.parse([]).has_error?
59
- assert_nil parse_result.parse_tree
25
+ def test_parse_result_has_error_result_if_only_end_token_passed
26
+ parse_result = SimpleParser.parse(build_tokens([Dhaka::END_SYMBOL_NAME]))
27
+ assert parse_result.has_error?
60
28
  end
61
29
 
62
30
  def test_parser_returns_error_result_with_index_of_bad_token_if_parse_error
63
- parse_result = CompiledArithmeticParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
31
+ parse_result = SimpleParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
64
32
  assert parse_result.has_error?
65
33
  assert_equal '-', parse_result.unexpected_token.symbol_name
66
34
  end
@@ -68,5 +36,4 @@ class TestCompiledParser < Test::Unit::TestCase
68
36
  def build_tokens(token_symbol_names)
69
37
  token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
70
38
  end
71
-
72
- end
39
+ end
@@ -0,0 +1,17 @@
1
+ unless defined? Dhaka
2
+ require File.dirname(__FILE__) + '/../lib/dhaka'
3
+ end
4
+
5
+ require File.dirname(__FILE__) + '/fake_logger'
6
+
7
+ require 'test/unit'
8
+
9
+ begin
10
+ require 'mocha'
11
+ rescue LoadError
12
+ puts "
13
+ The tests depend on Mocha. Please install it if you wish to run them:
14
+ sudo gem install mocha
15
+ This gem is not required for using the library."
16
+ exit
17
+ end
@@ -1,8 +1,23 @@
1
- require "test/unit"
2
- require File.dirname(__FILE__)+'/../lib/dhaka'
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
3
3
 
4
4
  class TestEvaluator < Test::Unit::TestCase
5
5
  def test_throws_exception_if_evaluation_rules_not_completely_defined
6
- assert_raise(Dhaka::EvaluatorDefinitionError) {require 'incomplete_arithmetic_evaluator'}
6
+ assert_raise(Dhaka::EvaluatorDefinitionError) do
7
+ eval(
8
+ "class IncompleteSimpleEvaluator < Dhaka::Evaluator
9
+ self.grammar = SimpleGrammar
10
+ define_evaluation_rules do
11
+ for_start do
12
+ something
13
+ end
14
+
15
+ for_literal do
16
+ something
17
+ end
18
+ end
19
+ end")
20
+ end
21
+
7
22
  end
8
23
  end
@@ -1,35 +1,32 @@
1
- #!/usr/bin/env ruby
2
- require 'test/unit'
3
- require 'simple_grammar'
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
4
3
 
5
4
  class SimpleGrammarTest < Test::Unit::TestCase
6
-
7
5
  def setup
8
6
  @grammar = SimpleGrammar
9
7
  end
10
8
 
11
9
  def test_loads_symbol_and_classifies_them
12
10
  expected_non_terminals = Set.new(['E', 'S', 'T', Dhaka::START_SYMBOL_NAME])
13
- expected_terminals = Set.new(['-', 'n', '(', ')', '#', Dhaka::END_SYMBOL_NAME])
11
+ expected_terminals = Set.new(['-', 'n', '(', ')', '#', Dhaka::END_SYMBOL_NAME])
14
12
  assert_equal(expected_non_terminals, Set.new(@grammar.non_terminal_symbols.collect {|symbol| symbol.name}))
15
13
  assert_equal(expected_terminals, Set.new(@grammar.terminal_symbols.collect {|symbol| symbol.name}))
16
14
  end
17
15
 
18
16
  def test_creates_productions
19
17
  productions_for_E = @grammar.productions_for_symbol(@grammar.symbol_for_name('E'))
20
- assert(productions_for_E.size==2)
18
+ assert(productions_for_E.size == 2)
21
19
  expected_productions_for_E = Set.new(['subtraction E ::= E - T', 'term E ::= T'])
22
20
  assert_equal(expected_productions_for_E, Set.new(productions_for_E.collect {|production| production.to_s}))
23
21
  productions_for_start = @grammar.productions_for_symbol(@grammar.start_symbol)
24
- assert(productions_for_start.size==1)
22
+ assert(productions_for_start.size == 1)
25
23
  expected_productions_for_start = Set.new(['start _Start_ ::= S #'])
26
24
  assert_equal(expected_productions_for_start, Set.new(productions_for_start.collect {|production| production.to_s}))
27
-
28
25
  end
29
26
 
30
27
  def test_symbols_in_productions_use_the_flyweight_pattern
31
28
  assert_same(@grammar.production_named('subtraction').symbol, @grammar.production_named('term').symbol)
32
- assert_same(@grammar.production_named('expression').expansion[0], @grammar.production_named('subtraction').expansion[0])
29
+ assert_same(@grammar.production_named('expression').expansion.first, @grammar.production_named('subtraction').expansion.first)
33
30
  end
34
31
 
35
32
  def test_first_with_non_terminal
@@ -43,11 +40,11 @@ class SimpleGrammarTest < Test::Unit::TestCase
43
40
  end
44
41
 
45
42
  def test_computes_closures_and_channels_given_a_kernel
46
- start_production = @grammar.production_named('start')
47
- start_item = Dhaka::Item.new(start_production, 0)
48
- kernel = Set.new([start_item])
43
+ start_production = @grammar.production_named('start')
44
+ start_item = Dhaka::Item.new(start_production, 0)
45
+ kernel = Set.new([start_item])
49
46
  channels, closure = @grammar.closure(kernel)
50
- expected_items = Set.new(['_Start_ ::= -> S # []',
47
+ expected_items = Set.new(['_Start_ ::= -> S # []',
51
48
  'S ::= -> E []',
52
49
  'E ::= -> E - T []',
53
50
  'E ::= -> T []',
@@ -65,6 +62,4 @@ class SimpleGrammarTest < Test::Unit::TestCase
65
62
  assert_equal(expected_items, Set.new(closure.values.collect{|item| item.to_s}))
66
63
  assert_equal(expected_channels, Set.new(channels.collect{|item| item.to_s}))
67
64
  end
68
-
69
-
70
65
  end
@@ -1,17 +1,19 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
-
3
1
  class LALRButNotSLRGrammar < Dhaka::Grammar
2
+
4
3
  for_symbol(Dhaka::START_SYMBOL_NAME) do
5
- start ['E']
4
+ start %w| E |
6
5
  end
6
+
7
7
  for_symbol('E') do
8
- E_Aa ['A', 'a']
9
- E_bAc ['b', 'A', 'c']
10
- E_dc ['d', 'c']
11
- E_bda ['b', 'd', 'a']
8
+ E_Aa %w| A a |
9
+ E_bAc %w| b A c |
10
+ E_dc %w| d c |
11
+ E_bda %w| b d a |
12
12
  end
13
+
13
14
  for_symbol('A') do
14
- A_d ['d']
15
+ A_d %w| d |
15
16
  end
17
+
16
18
  end
17
19
 
@@ -1,9 +1,7 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
-
3
1
  class MalformedGrammar < Dhaka::Grammar
4
2
 
5
- for_symbol('goo') do
6
- foo ['boo']
3
+ for_symbol('foo') do
4
+ bar %w| baz |
7
5
  end
8
6
 
9
7
  end
@@ -1,6 +1,5 @@
1
- require "test/unit"
2
-
3
- require "malformed_grammar"
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + "/malformed_grammar"
4
3
 
5
4
  class TestMalformedGrammar < Test::Unit::TestCase
6
5
  def test_must_have_a_start_symbol_in_order_to_generate_a_parser
@@ -1,18 +1,21 @@
1
- require File.dirname(__FILE__)+'/../lib/dhaka'
2
-
3
1
  class NullableGrammar < Dhaka::Grammar
2
+
4
3
  for_symbol(Dhaka::START_SYMBOL_NAME) do
5
- tuple ['Tuple']
4
+ tuple %w| Tuple |
6
5
  end
6
+
7
7
  for_symbol('Tuple') do
8
- element_list ['(', 'Elements', ')']
8
+ element_list %w| ( Elements ) |
9
9
  end
10
+
10
11
  for_symbol('Elements') do
11
- empty_element_list []
12
- concatenate_element_lists ['Character', 'Elements']
12
+ empty_element_list %w| |
13
+ concatenate_element_lists %w| Character Elements |
13
14
  end
15
+
14
16
  for_symbol('Character') do
15
- literal_a ['a']
16
- literal_b ['b']
17
+ literal_a %w| a |
18
+ literal_b %w| b |
17
19
  end
20
+
18
21
  end
@@ -0,0 +1,44 @@
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
3
+
4
+ class TestParseSuccessResult < Test::Unit::TestCase
5
+ include Dhaka
6
+
7
+ def composite_node(production, child_nodes, dot_name)
8
+ node = ParseTreeCompositeNode.new(SimpleGrammar.production_named(production))
9
+ node.child_nodes.concat child_nodes
10
+ node.stubs(:dot_name).returns(dot_name)
11
+ node
12
+ end
13
+
14
+ def leaf_node(token, value, dot_name)
15
+ node = ParseTreeLeafNode.new(Token.new(token, value, nil))
16
+ node.stubs(:dot_name).returns(dot_name)
17
+ node
18
+ end
19
+
20
+ def test_parse_tree_can_be_exported_to_dot_format
21
+ first_term = composite_node('literal', [leaf_node('n', 1, "literal_1")], "first_term")
22
+ second_term = composite_node('literal', [leaf_node('n', 2, "literal_2")], "second_term")
23
+ addition_term = leaf_node('-', nil, "subtraction_operator")
24
+ tree = composite_node('subtraction', [first_term, addition_term, second_term], "expression")
25
+ result = ParseSuccessResult.new(tree)
26
+ assert_equal(
27
+ %(digraph x {
28
+ node [fontsize="10" shape=box size="5"]
29
+ expression [label="subtraction E ::= E - T"]
30
+ expression -> first_term
31
+ first_term [label="literal T ::= n"]
32
+ first_term -> literal_1
33
+ literal_1 [label="n : 1"]
34
+ expression -> subtraction_operator
35
+ subtraction_operator [label="-"]
36
+ expression -> second_term
37
+ second_term [label="literal T ::= n"]
38
+ second_term -> literal_2
39
+ literal_2 [label="n : 2"]
40
+ }),
41
+ result.to_dot)
42
+ end
43
+
44
+ end
@@ -0,0 +1,36 @@
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+
3
+ class TestParserState < Test::Unit::TestCase
4
+ include Dhaka
5
+ def test_unique_identifier
6
+ states = (0...5).collect {|i| ParserState.new(nil, {})}
7
+ ids = Set.new
8
+ states.each do |state|
9
+ assert(/^State(\d+)$/ =~ state.unique_name)
10
+ ids << $1.to_i
11
+ end
12
+ assert_equal(5, ids.size)
13
+ end
14
+
15
+ def test_to_s_method
16
+ options = {:some_option => true}
17
+ item1 = mock()
18
+ item1.stubs(:to_s).with(options).returns("i'm item 1")
19
+ item2 = mock()
20
+ item2.stubs(:to_s).with(options).returns("i'm item 2")
21
+ state = ParserState.new(nil, {item1 => item1, item2 => item2})
22
+ assert_equal(["i'm item 1", "i'm item 2"], state.to_s(options).split("\n").sort)
23
+ end
24
+
25
+ def test_to_dot_method
26
+ options = {:some_option => true}
27
+ item1 = mock()
28
+ item1.stubs(:to_s).with(options).returns("i'm item 1")
29
+ item2 = mock()
30
+ item2.stubs(:to_s).with(options).returns("i'm item 2")
31
+ state = ParserState.new(nil, {item1 => item1, item2 => item2})
32
+ state.stubs(:unique_name).returns("State1")
33
+ assert(state.to_dot(options) =~ /^State1 \[label="(.+)\\n(.+)"\]/)
34
+ assert_equal(["i'm item 1", "i'm item 2"], [$1, $2].sort)
35
+ end
36
+ end
@@ -1,16 +1,12 @@
1
- #!/usr/bin/env ruby
2
- require 'test/unit'
3
- require 'simple_grammar'
4
- require 'arithmetic_grammar'
5
- require 'nullable_grammar'
6
- require 'lalr_but_not_slr_grammar'
7
- require 'another_lalr_but_not_slr_grammar'
8
- require 'rr_conflict_grammar'
9
- require 'sr_conflict_grammar'
10
- require 'fake_logger'
1
+ require File.dirname(__FILE__) + '/dhaka_test_helper'
2
+ require File.dirname(__FILE__) + '/simple_grammar'
3
+ require File.dirname(__FILE__) + '/nullable_grammar'
4
+ require File.dirname(__FILE__) + '/lalr_but_not_slr_grammar'
5
+ require File.dirname(__FILE__) + '/another_lalr_but_not_slr_grammar'
6
+ require File.dirname(__FILE__) + '/rr_conflict_grammar'
7
+ require File.dirname(__FILE__) + '/sr_conflict_grammar'
11
8
 
12
9
  class ParserTest < Test::Unit::TestCase
13
-
14
10
  def build_tokens(token_symbol_names)
15
11
  token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
16
12
  end
@@ -20,37 +16,45 @@ class ParserTest < Test::Unit::TestCase
20
16
  @parser = Dhaka::Parser.new(@grammar)
21
17
  end
22
18
 
19
+ def contains(given_member, set)
20
+ set.inject(false) {|result, member| result ||= (Set.new(member) == Set.new(given_member))}
21
+ end
22
+
23
+ def assert_collection_equal(expected, actual)
24
+ assert_equal(expected.size, actual.size)
25
+ actual.each do |actual_member|
26
+ assert(contains(actual_member, expected))
27
+ end
28
+ end
29
+
23
30
  def test_parser_generates_states_with_correct_items
24
- expected_states = {}
25
- expected_states[1] = Set.new(['_Start_ ::= -> S # [_End_]',
31
+ expected_states = [
32
+ ['_Start_ ::= -> S # [_End_]',
26
33
  'S ::= -> E [#]',
27
34
  'E ::= -> E - T [#-]',
28
35
  'E ::= -> T [#-]',
29
36
  'T ::= -> n [#-]',
30
- 'T ::= -> ( E ) [#-]'])
31
- expected_states[2] = Set.new(['E ::= T -> [#)-]'])
32
- expected_states[3] = Set.new(['T ::= n -> [#)-]'])
33
- expected_states[4] = Set.new(['S ::= E -> [#]',
34
- 'E ::= E -> - T [#-]'])
35
- expected_states[5] = Set.new(['_Start_ ::= S -> # [_End_]'])
36
- expected_states[6] = Set.new(['T ::= ( -> E ) [#)-]',
37
+ 'T ::= -> ( E ) [#-]'],
38
+ ['E ::= T -> [#)-]'],
39
+ ['T ::= n -> [#)-]'],
40
+ ['S ::= E -> [#]',
41
+ 'E ::= E -> - T [#-]'],
42
+ ['_Start_ ::= S -> # [_End_]'],
43
+ ['T ::= ( -> E ) [#)-]',
37
44
  'E ::= -> E - T [)-]',
38
45
  'E ::= -> T [)-]',
39
46
  'T ::= -> n [)-]',
40
- 'T ::= -> ( E ) [)-]'])
41
- expected_states[7] = Set.new(['E ::= E - -> T [#)-]',
47
+ 'T ::= -> ( E ) [)-]'],
48
+ ['E ::= E - -> T [#)-]',
42
49
  'T ::= -> n [#)-]',
43
- 'T ::= -> ( E ) [#)-]'])
44
- expected_states[8] = Set.new(['E ::= E - T -> [#)-]'])
45
- expected_states[9] = Set.new(['T ::= ( E -> ) [#)-]',
46
- 'E ::= E -> - T [)-]'])
47
- expected_states[10] = Set.new(['T ::= ( E ) -> [#)-]'])
48
- expected_states[11] = Set.new(['_Start_ ::= S # -> [_End_]'])
49
- actual_states = Set.new(@parser.send('states').collect {|state| Set.new(state.items.values.collect {|item| item.to_s})})
50
- #write_parser(@parser)
51
- expected_states.values.each do |state|
52
- assert set_finder(state, actual_states), "expected #{state.to_a}"
53
- end
50
+ 'T ::= -> ( E ) [#)-]'],
51
+ ['E ::= E - T -> [#)-]'],
52
+ ['T ::= ( E -> ) [#)-]',
53
+ 'E ::= E -> - T [)-]'],
54
+ ['T ::= ( E ) -> [#)-]'],
55
+ ['_Start_ ::= S # -> [_End_]']]
56
+ actual_states = @parser.send('states').collect {|state| state.items.values.collect {|item| item.to_s}}
57
+ assert_collection_equal(expected_states, actual_states)
54
58
  end
55
59
 
56
60
  def test_parser_can_be_exported_to_dot_format
@@ -58,7 +62,7 @@ class ParserTest < Test::Unit::TestCase
58
62
  end
59
63
 
60
64
  def test_parser_generates_parse_tree_given_a_stream_of_symbols
61
- parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME])).parse_tree
65
+ parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME]))
62
66
  assert_equal \
63
67
  ["literal",
64
68
  "term",
@@ -76,68 +80,13 @@ class ParserTest < Test::Unit::TestCase
76
80
  "start"], parse_tree.linearize.collect {|node| node.production.name}
77
81
  end
78
82
 
79
- def test_parse_trees_can_be_exported_to_dot_format
80
- parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME])).parse_tree
81
- parse_tree.to_dot
82
- end
83
-
84
83
  def get_linearized_parse_result(input, parser)
85
84
  parser.parse(build_tokens(input)).parse_tree.linearize.collect {|node| node.production.name}
86
85
  end
87
86
 
88
- def test_with_a_different_grammar_with_division
89
- grammar = ArithmeticGrammar
90
- parser = Dhaka::Parser.new(grammar)
91
- parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')', Dhaka::END_SYMBOL_NAME]
92
- assert_equal \
93
- ["getting_literals",
94
- "factor",
95
- "term",
96
- "getting_literals",
97
- "factor",
98
- "getting_literals",
99
- "division",
100
- "term",
101
- "getting_literals",
102
- "factor",
103
- "subtraction",
104
- "unpacking_parenthetized_expression",
105
- "factor",
106
- "getting_literals",
107
- "division",
108
- "subtraction",
109
- "unpacking_parenthetized_expression",
110
- "factor",
111
- "term",
112
- "expression"], get_linearized_parse_result(parser_input, parser)
113
-
114
- parser_input = ['h','(','(','n',')','-','n',',','n',')', Dhaka::END_SYMBOL_NAME]
115
- assert_equal \
116
- ["max_function",
117
- "getting_literals",
118
- "factor",
119
- "term",
120
- "unpacking_parenthetized_expression",
121
- "factor",
122
- "term",
123
- "getting_literals",
124
- "factor",
125
- "subtraction",
126
- "getting_literals",
127
- "factor",
128
- "term",
129
- "single_args",
130
- "concatenating_args",
131
- "evaluating_function",
132
- "function",
133
- "factor",
134
- "term",
135
- "expression"], get_linearized_parse_result(parser_input, parser)
136
- end
137
-
138
87
  def test_with_a_grammar_with_nullables_after_terminals
139
- grammar = NullableGrammar
140
- parser = Dhaka::Parser.new(grammar)
88
+ grammar = NullableGrammar
89
+ parser = Dhaka::Parser.new(grammar)
141
90
  parser_input = ['(','a',')', Dhaka::END_SYMBOL_NAME]
142
91
  assert_equal \
143
92
  ["literal_a",
@@ -148,30 +97,31 @@ class ParserTest < Test::Unit::TestCase
148
97
  end
149
98
 
150
99
  def test_with_a_grammar_that_is_not_SLR
151
- grammar = LALRButNotSLRGrammar
152
- parser = Dhaka::Parser.new(grammar)
100
+ grammar = LALRButNotSLRGrammar
101
+ parser = Dhaka::Parser.new(grammar)
153
102
  parser_input = ['b','d','c', Dhaka::END_SYMBOL_NAME]
154
103
  assert_equal(["A_d", "E_bAc", "start"], get_linearized_parse_result(parser_input, parser))
155
104
  end
156
105
 
157
106
  def test_with_another_grammar_that_is_not_SLR
158
- grammar = AnotherLALRButNotSLRGrammar
159
- parser = Dhaka::Parser.new(grammar)
107
+ grammar = AnotherLALRButNotSLRGrammar
108
+ parser = Dhaka::Parser.new(grammar)
160
109
  parser_input = ['*', 'id', '=', 'id', Dhaka::END_SYMBOL_NAME]
161
110
  assert_equal(["identifier", "l_value", "contents", "identifier", "l_value", "assignment"], get_linearized_parse_result(parser_input, parser))
162
111
  end
163
112
 
164
- def test_with_a_grammar_that_should_generate_an_RR_conflict
165
- grammar = RRConflictGrammar
113
+ def test_debug_output_with_a_grammar_that_should_generate_an_RR_conflict
166
114
  fake_logger = FakeLogger.new
167
- Dhaka::Parser.new(grammar, fake_logger)
168
- assert_equal(1, fake_logger.errors.size)
115
+ Dhaka::ParserState.any_instance.stubs(:unique_name).returns("StateXXX")
116
+ parser = Dhaka::Parser.new(RRConflictGrammar, fake_logger)
117
+ num_states = parser.send(:states).size
118
+ assert_equal(['Created StateXXX.'] * num_states, fake_logger.debugs[0...num_states])
119
+ assert_equal(1, fake_logger.errors.size)
120
+ assert(fake_logger.errors.first.match(/^Parser Conflict at State:\n(.+)\n(.+)\nExisting: (.+)\nNew: (.+)\nLookahead: (.+)$/))
121
+ assert_equal(["A ::= x y ->", "B ::= x y ->", "Reduce with xy A ::= x y", "Reduce with xy_again B ::= x y", "c"],
122
+ $~[1..5].sort)
169
123
  end
170
124
 
171
- def set_finder(set1, set2)
172
- set2.inject(false) {|result, member| result ||= member == set1}
173
- end
174
-
175
125
  def write_parser(parser)
176
126
  File.open('parser.dot', 'w') do |file|
177
127
  file << parser.to_dot