dhaka 2.0.0 → 2.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/dhaka.rb +24 -22
- data/lib/evaluator/evaluator.rb +42 -44
- data/lib/grammar/closure_hash.rb +4 -3
- data/lib/grammar/grammar.rb +113 -110
- data/lib/grammar/grammar_symbol.rb +6 -3
- data/lib/grammar/precedence.rb +3 -2
- data/lib/grammar/production.rb +5 -6
- data/lib/parser/action.rb +16 -11
- data/lib/parser/channel.rb +22 -16
- data/lib/parser/compiled_parser.rb +28 -22
- data/lib/parser/conflict.rb +54 -0
- data/lib/parser/item.rb +19 -19
- data/lib/parser/parse_result.rb +16 -1
- data/lib/parser/parse_tree.rb +15 -9
- data/lib/parser/parser.rb +51 -80
- data/lib/parser/parser_run.rb +6 -6
- data/lib/parser/parser_state.rb +16 -18
- data/lib/parser/token.rb +6 -4
- data/lib/tokenizer/tokenizer.rb +34 -31
- data/test/all_tests.rb +4 -18
- data/test/another_lalr_but_not_slr_grammar.rb +9 -5
- data/test/{arithmetic_evaluator.rb → arithmetic/arithmetic_evaluator.rb} +1 -2
- data/test/{arithmetic_evaluator_test.rb → arithmetic/arithmetic_evaluator_test.rb} +9 -20
- data/test/arithmetic/arithmetic_grammar.rb +41 -0
- data/test/{arithmetic_grammar_test.rb → arithmetic/arithmetic_grammar_test.rb} +2 -4
- data/test/{arithmetic_test_methods.rb → arithmetic/arithmetic_test_methods.rb} +1 -3
- data/test/{arithmetic_tokenizer.rb → arithmetic/arithmetic_tokenizer.rb} +8 -10
- data/test/{arithmetic_tokenizer_test.rb → arithmetic/arithmetic_tokenizer_test.rb} +4 -2
- data/test/{arithmetic_precedence_evaluator.rb → arithmetic_precedence/arithmetic_precedence_evaluator.rb} +1 -2
- data/test/arithmetic_precedence/arithmetic_precedence_grammar.rb +24 -0
- data/test/{arithmetic_precedence_grammar_test.rb → arithmetic_precedence/arithmetic_precedence_grammar_test.rb} +2 -3
- data/test/arithmetic_precedence/arithmetic_precedence_parser_test.rb +31 -0
- data/test/{arithmetic_precedence_tokenizer.rb → arithmetic_precedence/arithmetic_precedence_tokenizer.rb} +8 -10
- data/test/brackets/bracket_grammar.rb +23 -0
- data/test/{bracket_tokenizer.rb → brackets/bracket_tokenizer.rb} +2 -4
- data/test/{brackets_test.rb → brackets/brackets_test.rb} +3 -4
- data/test/chittagong/chittagong_driver.rb +47 -0
- data/test/{chittagong_driver_test.rb → chittagong/chittagong_driver_test.rb} +66 -58
- data/test/{chittagong_evaluator.rb → chittagong/chittagong_evaluator.rb} +28 -13
- data/test/{chittagong_evaluator_test.rb → chittagong/chittagong_evaluator_test.rb} +6 -10
- data/test/chittagong/chittagong_grammar.rb +110 -0
- data/test/{chittagong_parser_test.rb → chittagong/chittagong_parser_test.rb} +5 -7
- data/test/{chittagong_test.rb → chittagong/chittagong_test.rb} +27 -36
- data/test/{chittagong_tokenizer.rb → chittagong/chittagong_tokenizer.rb} +17 -17
- data/test/{chittagong_tokenizer_test.rb → chittagong/chittagong_tokenizer_test.rb} +2 -3
- data/test/compiled_parser_test.rb +9 -42
- data/test/dhaka_test_helper.rb +17 -0
- data/test/evaluator_test.rb +18 -3
- data/test/grammar_test.rb +10 -15
- data/test/lalr_but_not_slr_grammar.rb +10 -8
- data/test/malformed_grammar.rb +2 -4
- data/test/malformed_grammar_test.rb +2 -3
- data/test/nullable_grammar.rb +11 -8
- data/test/parse_result_test.rb +44 -0
- data/test/parser_state_test.rb +36 -0
- data/test/parser_test.rb +53 -103
- data/test/precedence_grammar.rb +6 -6
- data/test/precedence_grammar_test.rb +2 -3
- data/test/rr_conflict_grammar.rb +5 -7
- data/test/simple_grammar.rb +6 -8
- data/test/sr_conflict_grammar.rb +6 -6
- metadata +30 -26
- data/test/arithmetic_grammar.rb +0 -35
- data/test/arithmetic_precedence_grammar.rb +0 -24
- data/test/arithmetic_precedence_parser_test.rb +0 -33
- data/test/bracket_grammar.rb +0 -25
- data/test/chittagong_grammar.rb +0 -104
- data/test/incomplete_arithmetic_evaluator.rb +0 -60
@@ -1,21 +1,20 @@
|
|
1
|
-
require File.dirname(__FILE__)+'
|
2
|
-
require 'chittagong_grammar'
|
3
|
-
|
1
|
+
require File.dirname(__FILE__) + '/chittagong_grammar'
|
4
2
|
|
5
3
|
class ChittagongTokenizer < Dhaka::Tokenizer
|
6
4
|
|
7
|
-
KEYWORDS
|
5
|
+
KEYWORDS = %w| print if else end while def return |
|
8
6
|
|
9
|
-
digits
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
7
|
+
digits = ('0'..'9').to_a
|
8
|
+
decimal_point = %w| . |
|
9
|
+
letters = ('a'..'z').to_a
|
10
|
+
parenths = %w| ( ) |
|
11
|
+
operators = %w| - + / * ^ ! > < |
|
12
|
+
equal_sign = %w| = |
|
13
|
+
whitespace = [' ']
|
14
|
+
arg_separator = %w| , |
|
15
|
+
newline = ["\n"]
|
17
16
|
|
18
|
-
all_characters = digits + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
|
17
|
+
all_characters = digits + decimal_point + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
|
19
18
|
|
20
19
|
for_state Dhaka::TOKENIZER_IDLE_STATE do
|
21
20
|
for_characters(all_characters - (digits + letters + newline + whitespace + equal_sign)) do
|
@@ -49,7 +48,7 @@ class ChittagongTokenizer < Dhaka::Tokenizer
|
|
49
48
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
50
49
|
end
|
51
50
|
for_character equal_sign do
|
52
|
-
curr_token.symbol_name
|
51
|
+
curr_token.symbol_name << '='
|
53
52
|
advance
|
54
53
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
55
54
|
end
|
@@ -60,18 +59,18 @@ class ChittagongTokenizer < Dhaka::Tokenizer
|
|
60
59
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
61
60
|
end
|
62
61
|
for_characters digits do
|
63
|
-
curr_token.value
|
62
|
+
curr_token.value << curr_char
|
64
63
|
advance
|
65
64
|
end
|
66
65
|
end
|
67
|
-
|
66
|
+
# lipi:keywords
|
68
67
|
for_state :get_word_literal do
|
69
68
|
for_characters all_characters - letters do
|
70
69
|
curr_token.symbol_name = word_literal_symbol(curr_token.value)
|
71
70
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
72
71
|
end
|
73
72
|
for_characters letters do
|
74
|
-
curr_token.value
|
73
|
+
curr_token.value << curr_char
|
75
74
|
advance
|
76
75
|
curr_token.symbol_name = word_literal_symbol(curr_token.value) unless curr_char
|
77
76
|
end
|
@@ -84,5 +83,6 @@ class ChittagongTokenizer < Dhaka::Tokenizer
|
|
84
83
|
'word_literal'
|
85
84
|
end
|
86
85
|
end
|
86
|
+
# lipi:keywords
|
87
87
|
|
88
88
|
end
|
@@ -1,16 +1,10 @@
|
|
1
|
-
require
|
2
|
-
require "simple_grammar"
|
3
|
-
|
4
|
-
require 'arithmetic_grammar'
|
5
|
-
|
6
|
-
eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as('Foo'))
|
7
|
-
eval(Dhaka::Parser.new(ArithmeticGrammar).compile_to_ruby_source_as('CompiledArithmeticParser'))
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + "/simple_grammar"
|
3
|
+
eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as(:SimpleParser))
|
8
4
|
|
9
5
|
class TestCompiledParser < Test::Unit::TestCase
|
10
|
-
include ArithmeticTestMethods
|
11
|
-
|
12
6
|
def test_compiled_parser_generates_parse_tree_for_simple_grammar
|
13
|
-
parse_tree =
|
7
|
+
parse_tree = SimpleParser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME]))
|
14
8
|
assert_equal \
|
15
9
|
["literal",
|
16
10
|
"term",
|
@@ -28,39 +22,13 @@ class TestCompiledParser < Test::Unit::TestCase
|
|
28
22
|
"start"], parse_tree.linearize.collect {|node| node.production.name}
|
29
23
|
end
|
30
24
|
|
31
|
-
def
|
32
|
-
|
33
|
-
|
34
|
-
["getting_literals",
|
35
|
-
"factor",
|
36
|
-
"term",
|
37
|
-
"getting_literals",
|
38
|
-
"factor",
|
39
|
-
"getting_literals",
|
40
|
-
"division",
|
41
|
-
"term",
|
42
|
-
"getting_literals",
|
43
|
-
"factor",
|
44
|
-
"subtraction",
|
45
|
-
"unpacking_parenthetized_expression",
|
46
|
-
"factor",
|
47
|
-
"getting_literals",
|
48
|
-
"division",
|
49
|
-
"subtraction",
|
50
|
-
"unpacking_parenthetized_expression",
|
51
|
-
"factor",
|
52
|
-
"term",
|
53
|
-
"expression"], parse(build_tokens(parser_input)).linearize.collect {|node| node.production.name}
|
54
|
-
end
|
55
|
-
|
56
|
-
def test_parse_result_has_nil_parse_tree_if_empty_token_array
|
57
|
-
parse_result = CompiledArithmeticParser.parse([])
|
58
|
-
assert !CompiledArithmeticParser.parse([]).has_error?
|
59
|
-
assert_nil parse_result.parse_tree
|
25
|
+
def test_parse_result_has_error_result_if_only_end_token_passed
|
26
|
+
parse_result = SimpleParser.parse(build_tokens([Dhaka::END_SYMBOL_NAME]))
|
27
|
+
assert parse_result.has_error?
|
60
28
|
end
|
61
29
|
|
62
30
|
def test_parser_returns_error_result_with_index_of_bad_token_if_parse_error
|
63
|
-
parse_result =
|
31
|
+
parse_result = SimpleParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
|
64
32
|
assert parse_result.has_error?
|
65
33
|
assert_equal '-', parse_result.unexpected_token.symbol_name
|
66
34
|
end
|
@@ -68,5 +36,4 @@ class TestCompiledParser < Test::Unit::TestCase
|
|
68
36
|
def build_tokens(token_symbol_names)
|
69
37
|
token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
|
70
38
|
end
|
71
|
-
|
72
|
-
end
|
39
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
unless defined? Dhaka
|
2
|
+
require File.dirname(__FILE__) + '/../lib/dhaka'
|
3
|
+
end
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/fake_logger'
|
6
|
+
|
7
|
+
require 'test/unit'
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'mocha'
|
11
|
+
rescue LoadError
|
12
|
+
puts "
|
13
|
+
The tests depend on Mocha. Please install it if you wish to run them:
|
14
|
+
sudo gem install mocha
|
15
|
+
This gem is not required for using the library."
|
16
|
+
exit
|
17
|
+
end
|
data/test/evaluator_test.rb
CHANGED
@@ -1,8 +1,23 @@
|
|
1
|
-
require
|
2
|
-
require File.dirname(__FILE__)+'
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
3
3
|
|
4
4
|
class TestEvaluator < Test::Unit::TestCase
|
5
5
|
def test_throws_exception_if_evaluation_rules_not_completely_defined
|
6
|
-
assert_raise(Dhaka::EvaluatorDefinitionError)
|
6
|
+
assert_raise(Dhaka::EvaluatorDefinitionError) do
|
7
|
+
eval(
|
8
|
+
"class IncompleteSimpleEvaluator < Dhaka::Evaluator
|
9
|
+
self.grammar = SimpleGrammar
|
10
|
+
define_evaluation_rules do
|
11
|
+
for_start do
|
12
|
+
something
|
13
|
+
end
|
14
|
+
|
15
|
+
for_literal do
|
16
|
+
something
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end")
|
20
|
+
end
|
21
|
+
|
7
22
|
end
|
8
23
|
end
|
data/test/grammar_test.rb
CHANGED
@@ -1,35 +1,32 @@
|
|
1
|
-
|
2
|
-
require '
|
3
|
-
require 'simple_grammar'
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
4
3
|
|
5
4
|
class SimpleGrammarTest < Test::Unit::TestCase
|
6
|
-
|
7
5
|
def setup
|
8
6
|
@grammar = SimpleGrammar
|
9
7
|
end
|
10
8
|
|
11
9
|
def test_loads_symbol_and_classifies_them
|
12
10
|
expected_non_terminals = Set.new(['E', 'S', 'T', Dhaka::START_SYMBOL_NAME])
|
13
|
-
expected_terminals
|
11
|
+
expected_terminals = Set.new(['-', 'n', '(', ')', '#', Dhaka::END_SYMBOL_NAME])
|
14
12
|
assert_equal(expected_non_terminals, Set.new(@grammar.non_terminal_symbols.collect {|symbol| symbol.name}))
|
15
13
|
assert_equal(expected_terminals, Set.new(@grammar.terminal_symbols.collect {|symbol| symbol.name}))
|
16
14
|
end
|
17
15
|
|
18
16
|
def test_creates_productions
|
19
17
|
productions_for_E = @grammar.productions_for_symbol(@grammar.symbol_for_name('E'))
|
20
|
-
assert(productions_for_E.size==2)
|
18
|
+
assert(productions_for_E.size == 2)
|
21
19
|
expected_productions_for_E = Set.new(['subtraction E ::= E - T', 'term E ::= T'])
|
22
20
|
assert_equal(expected_productions_for_E, Set.new(productions_for_E.collect {|production| production.to_s}))
|
23
21
|
productions_for_start = @grammar.productions_for_symbol(@grammar.start_symbol)
|
24
|
-
assert(productions_for_start.size==1)
|
22
|
+
assert(productions_for_start.size == 1)
|
25
23
|
expected_productions_for_start = Set.new(['start _Start_ ::= S #'])
|
26
24
|
assert_equal(expected_productions_for_start, Set.new(productions_for_start.collect {|production| production.to_s}))
|
27
|
-
|
28
25
|
end
|
29
26
|
|
30
27
|
def test_symbols_in_productions_use_the_flyweight_pattern
|
31
28
|
assert_same(@grammar.production_named('subtraction').symbol, @grammar.production_named('term').symbol)
|
32
|
-
assert_same(@grammar.production_named('expression').expansion
|
29
|
+
assert_same(@grammar.production_named('expression').expansion.first, @grammar.production_named('subtraction').expansion.first)
|
33
30
|
end
|
34
31
|
|
35
32
|
def test_first_with_non_terminal
|
@@ -43,11 +40,11 @@ class SimpleGrammarTest < Test::Unit::TestCase
|
|
43
40
|
end
|
44
41
|
|
45
42
|
def test_computes_closures_and_channels_given_a_kernel
|
46
|
-
start_production
|
47
|
-
start_item
|
48
|
-
kernel
|
43
|
+
start_production = @grammar.production_named('start')
|
44
|
+
start_item = Dhaka::Item.new(start_production, 0)
|
45
|
+
kernel = Set.new([start_item])
|
49
46
|
channels, closure = @grammar.closure(kernel)
|
50
|
-
expected_items
|
47
|
+
expected_items = Set.new(['_Start_ ::= -> S # []',
|
51
48
|
'S ::= -> E []',
|
52
49
|
'E ::= -> E - T []',
|
53
50
|
'E ::= -> T []',
|
@@ -65,6 +62,4 @@ class SimpleGrammarTest < Test::Unit::TestCase
|
|
65
62
|
assert_equal(expected_items, Set.new(closure.values.collect{|item| item.to_s}))
|
66
63
|
assert_equal(expected_channels, Set.new(channels.collect{|item| item.to_s}))
|
67
64
|
end
|
68
|
-
|
69
|
-
|
70
65
|
end
|
@@ -1,17 +1,19 @@
|
|
1
|
-
require File.dirname(__FILE__)+'/../lib/dhaka'
|
2
|
-
|
3
1
|
class LALRButNotSLRGrammar < Dhaka::Grammar
|
2
|
+
|
4
3
|
for_symbol(Dhaka::START_SYMBOL_NAME) do
|
5
|
-
start
|
4
|
+
start %w| E |
|
6
5
|
end
|
6
|
+
|
7
7
|
for_symbol('E') do
|
8
|
-
E_Aa
|
9
|
-
E_bAc
|
10
|
-
E_dc
|
11
|
-
E_bda
|
8
|
+
E_Aa %w| A a |
|
9
|
+
E_bAc %w| b A c |
|
10
|
+
E_dc %w| d c |
|
11
|
+
E_bda %w| b d a |
|
12
12
|
end
|
13
|
+
|
13
14
|
for_symbol('A') do
|
14
|
-
A_d
|
15
|
+
A_d %w| d |
|
15
16
|
end
|
17
|
+
|
16
18
|
end
|
17
19
|
|
data/test/malformed_grammar.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
|
-
require
|
2
|
-
|
3
|
-
require "malformed_grammar"
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + "/malformed_grammar"
|
4
3
|
|
5
4
|
class TestMalformedGrammar < Test::Unit::TestCase
|
6
5
|
def test_must_have_a_start_symbol_in_order_to_generate_a_parser
|
data/test/nullable_grammar.rb
CHANGED
@@ -1,18 +1,21 @@
|
|
1
|
-
require File.dirname(__FILE__)+'/../lib/dhaka'
|
2
|
-
|
3
1
|
class NullableGrammar < Dhaka::Grammar
|
2
|
+
|
4
3
|
for_symbol(Dhaka::START_SYMBOL_NAME) do
|
5
|
-
tuple
|
4
|
+
tuple %w| Tuple |
|
6
5
|
end
|
6
|
+
|
7
7
|
for_symbol('Tuple') do
|
8
|
-
element_list
|
8
|
+
element_list %w| ( Elements ) |
|
9
9
|
end
|
10
|
+
|
10
11
|
for_symbol('Elements') do
|
11
|
-
empty_element_list
|
12
|
-
concatenate_element_lists
|
12
|
+
empty_element_list %w| |
|
13
|
+
concatenate_element_lists %w| Character Elements |
|
13
14
|
end
|
15
|
+
|
14
16
|
for_symbol('Character') do
|
15
|
-
literal_a
|
16
|
-
literal_b
|
17
|
+
literal_a %w| a |
|
18
|
+
literal_b %w| b |
|
17
19
|
end
|
20
|
+
|
18
21
|
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
3
|
+
|
4
|
+
class TestParseSuccessResult < Test::Unit::TestCase
|
5
|
+
include Dhaka
|
6
|
+
|
7
|
+
def composite_node(production, child_nodes, dot_name)
|
8
|
+
node = ParseTreeCompositeNode.new(SimpleGrammar.production_named(production))
|
9
|
+
node.child_nodes.concat child_nodes
|
10
|
+
node.stubs(:dot_name).returns(dot_name)
|
11
|
+
node
|
12
|
+
end
|
13
|
+
|
14
|
+
def leaf_node(token, value, dot_name)
|
15
|
+
node = ParseTreeLeafNode.new(Token.new(token, value, nil))
|
16
|
+
node.stubs(:dot_name).returns(dot_name)
|
17
|
+
node
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_parse_tree_can_be_exported_to_dot_format
|
21
|
+
first_term = composite_node('literal', [leaf_node('n', 1, "literal_1")], "first_term")
|
22
|
+
second_term = composite_node('literal', [leaf_node('n', 2, "literal_2")], "second_term")
|
23
|
+
addition_term = leaf_node('-', nil, "subtraction_operator")
|
24
|
+
tree = composite_node('subtraction', [first_term, addition_term, second_term], "expression")
|
25
|
+
result = ParseSuccessResult.new(tree)
|
26
|
+
assert_equal(
|
27
|
+
%(digraph x {
|
28
|
+
node [fontsize="10" shape=box size="5"]
|
29
|
+
expression [label="subtraction E ::= E - T"]
|
30
|
+
expression -> first_term
|
31
|
+
first_term [label="literal T ::= n"]
|
32
|
+
first_term -> literal_1
|
33
|
+
literal_1 [label="n : 1"]
|
34
|
+
expression -> subtraction_operator
|
35
|
+
subtraction_operator [label="-"]
|
36
|
+
expression -> second_term
|
37
|
+
second_term [label="literal T ::= n"]
|
38
|
+
second_term -> literal_2
|
39
|
+
literal_2 [label="n : 2"]
|
40
|
+
}),
|
41
|
+
result.to_dot)
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
|
3
|
+
class TestParserState < Test::Unit::TestCase
|
4
|
+
include Dhaka
|
5
|
+
def test_unique_identifier
|
6
|
+
states = (0...5).collect {|i| ParserState.new(nil, {})}
|
7
|
+
ids = Set.new
|
8
|
+
states.each do |state|
|
9
|
+
assert(/^State(\d+)$/ =~ state.unique_name)
|
10
|
+
ids << $1.to_i
|
11
|
+
end
|
12
|
+
assert_equal(5, ids.size)
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_to_s_method
|
16
|
+
options = {:some_option => true}
|
17
|
+
item1 = mock()
|
18
|
+
item1.stubs(:to_s).with(options).returns("i'm item 1")
|
19
|
+
item2 = mock()
|
20
|
+
item2.stubs(:to_s).with(options).returns("i'm item 2")
|
21
|
+
state = ParserState.new(nil, {item1 => item1, item2 => item2})
|
22
|
+
assert_equal(["i'm item 1", "i'm item 2"], state.to_s(options).split("\n").sort)
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_to_dot_method
|
26
|
+
options = {:some_option => true}
|
27
|
+
item1 = mock()
|
28
|
+
item1.stubs(:to_s).with(options).returns("i'm item 1")
|
29
|
+
item2 = mock()
|
30
|
+
item2.stubs(:to_s).with(options).returns("i'm item 2")
|
31
|
+
state = ParserState.new(nil, {item1 => item1, item2 => item2})
|
32
|
+
state.stubs(:unique_name).returns("State1")
|
33
|
+
assert(state.to_dot(options) =~ /^State1 \[label="(.+)\\n(.+)"\]/)
|
34
|
+
assert_equal(["i'm item 1", "i'm item 2"], [$1, $2].sort)
|
35
|
+
end
|
36
|
+
end
|
data/test/parser_test.rb
CHANGED
@@ -1,16 +1,12 @@
|
|
1
|
-
|
2
|
-
require '
|
3
|
-
require '
|
4
|
-
require '
|
5
|
-
require '
|
6
|
-
require '
|
7
|
-
require '
|
8
|
-
require 'rr_conflict_grammar'
|
9
|
-
require 'sr_conflict_grammar'
|
10
|
-
require 'fake_logger'
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
3
|
+
require File.dirname(__FILE__) + '/nullable_grammar'
|
4
|
+
require File.dirname(__FILE__) + '/lalr_but_not_slr_grammar'
|
5
|
+
require File.dirname(__FILE__) + '/another_lalr_but_not_slr_grammar'
|
6
|
+
require File.dirname(__FILE__) + '/rr_conflict_grammar'
|
7
|
+
require File.dirname(__FILE__) + '/sr_conflict_grammar'
|
11
8
|
|
12
9
|
class ParserTest < Test::Unit::TestCase
|
13
|
-
|
14
10
|
def build_tokens(token_symbol_names)
|
15
11
|
token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
|
16
12
|
end
|
@@ -20,37 +16,45 @@ class ParserTest < Test::Unit::TestCase
|
|
20
16
|
@parser = Dhaka::Parser.new(@grammar)
|
21
17
|
end
|
22
18
|
|
19
|
+
def contains(given_member, set)
|
20
|
+
set.inject(false) {|result, member| result ||= (Set.new(member) == Set.new(given_member))}
|
21
|
+
end
|
22
|
+
|
23
|
+
def assert_collection_equal(expected, actual)
|
24
|
+
assert_equal(expected.size, actual.size)
|
25
|
+
actual.each do |actual_member|
|
26
|
+
assert(contains(actual_member, expected))
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
23
30
|
def test_parser_generates_states_with_correct_items
|
24
|
-
expected_states =
|
25
|
-
|
31
|
+
expected_states = [
|
32
|
+
['_Start_ ::= -> S # [_End_]',
|
26
33
|
'S ::= -> E [#]',
|
27
34
|
'E ::= -> E - T [#-]',
|
28
35
|
'E ::= -> T [#-]',
|
29
36
|
'T ::= -> n [#-]',
|
30
|
-
'T ::= -> ( E ) [#-]']
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
'E ::= E -> - T [#-]']
|
35
|
-
|
36
|
-
|
37
|
+
'T ::= -> ( E ) [#-]'],
|
38
|
+
['E ::= T -> [#)-]'],
|
39
|
+
['T ::= n -> [#)-]'],
|
40
|
+
['S ::= E -> [#]',
|
41
|
+
'E ::= E -> - T [#-]'],
|
42
|
+
['_Start_ ::= S -> # [_End_]'],
|
43
|
+
['T ::= ( -> E ) [#)-]',
|
37
44
|
'E ::= -> E - T [)-]',
|
38
45
|
'E ::= -> T [)-]',
|
39
46
|
'T ::= -> n [)-]',
|
40
|
-
'T ::= -> ( E ) [)-]']
|
41
|
-
|
47
|
+
'T ::= -> ( E ) [)-]'],
|
48
|
+
['E ::= E - -> T [#)-]',
|
42
49
|
'T ::= -> n [#)-]',
|
43
|
-
'T ::= -> ( E ) [#)-]']
|
44
|
-
|
45
|
-
|
46
|
-
'E ::= E -> - T [)-]']
|
47
|
-
|
48
|
-
|
49
|
-
actual_states =
|
50
|
-
|
51
|
-
expected_states.values.each do |state|
|
52
|
-
assert set_finder(state, actual_states), "expected #{state.to_a}"
|
53
|
-
end
|
50
|
+
'T ::= -> ( E ) [#)-]'],
|
51
|
+
['E ::= E - T -> [#)-]'],
|
52
|
+
['T ::= ( E -> ) [#)-]',
|
53
|
+
'E ::= E -> - T [)-]'],
|
54
|
+
['T ::= ( E ) -> [#)-]'],
|
55
|
+
['_Start_ ::= S # -> [_End_]']]
|
56
|
+
actual_states = @parser.send('states').collect {|state| state.items.values.collect {|item| item.to_s}}
|
57
|
+
assert_collection_equal(expected_states, actual_states)
|
54
58
|
end
|
55
59
|
|
56
60
|
def test_parser_can_be_exported_to_dot_format
|
@@ -58,7 +62,7 @@ class ParserTest < Test::Unit::TestCase
|
|
58
62
|
end
|
59
63
|
|
60
64
|
def test_parser_generates_parse_tree_given_a_stream_of_symbols
|
61
|
-
parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME]))
|
65
|
+
parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME]))
|
62
66
|
assert_equal \
|
63
67
|
["literal",
|
64
68
|
"term",
|
@@ -76,68 +80,13 @@ class ParserTest < Test::Unit::TestCase
|
|
76
80
|
"start"], parse_tree.linearize.collect {|node| node.production.name}
|
77
81
|
end
|
78
82
|
|
79
|
-
def test_parse_trees_can_be_exported_to_dot_format
|
80
|
-
parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME])).parse_tree
|
81
|
-
parse_tree.to_dot
|
82
|
-
end
|
83
|
-
|
84
83
|
def get_linearized_parse_result(input, parser)
|
85
84
|
parser.parse(build_tokens(input)).parse_tree.linearize.collect {|node| node.production.name}
|
86
85
|
end
|
87
86
|
|
88
|
-
def test_with_a_different_grammar_with_division
|
89
|
-
grammar = ArithmeticGrammar
|
90
|
-
parser = Dhaka::Parser.new(grammar)
|
91
|
-
parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')', Dhaka::END_SYMBOL_NAME]
|
92
|
-
assert_equal \
|
93
|
-
["getting_literals",
|
94
|
-
"factor",
|
95
|
-
"term",
|
96
|
-
"getting_literals",
|
97
|
-
"factor",
|
98
|
-
"getting_literals",
|
99
|
-
"division",
|
100
|
-
"term",
|
101
|
-
"getting_literals",
|
102
|
-
"factor",
|
103
|
-
"subtraction",
|
104
|
-
"unpacking_parenthetized_expression",
|
105
|
-
"factor",
|
106
|
-
"getting_literals",
|
107
|
-
"division",
|
108
|
-
"subtraction",
|
109
|
-
"unpacking_parenthetized_expression",
|
110
|
-
"factor",
|
111
|
-
"term",
|
112
|
-
"expression"], get_linearized_parse_result(parser_input, parser)
|
113
|
-
|
114
|
-
parser_input = ['h','(','(','n',')','-','n',',','n',')', Dhaka::END_SYMBOL_NAME]
|
115
|
-
assert_equal \
|
116
|
-
["max_function",
|
117
|
-
"getting_literals",
|
118
|
-
"factor",
|
119
|
-
"term",
|
120
|
-
"unpacking_parenthetized_expression",
|
121
|
-
"factor",
|
122
|
-
"term",
|
123
|
-
"getting_literals",
|
124
|
-
"factor",
|
125
|
-
"subtraction",
|
126
|
-
"getting_literals",
|
127
|
-
"factor",
|
128
|
-
"term",
|
129
|
-
"single_args",
|
130
|
-
"concatenating_args",
|
131
|
-
"evaluating_function",
|
132
|
-
"function",
|
133
|
-
"factor",
|
134
|
-
"term",
|
135
|
-
"expression"], get_linearized_parse_result(parser_input, parser)
|
136
|
-
end
|
137
|
-
|
138
87
|
def test_with_a_grammar_with_nullables_after_terminals
|
139
|
-
grammar
|
140
|
-
parser
|
88
|
+
grammar = NullableGrammar
|
89
|
+
parser = Dhaka::Parser.new(grammar)
|
141
90
|
parser_input = ['(','a',')', Dhaka::END_SYMBOL_NAME]
|
142
91
|
assert_equal \
|
143
92
|
["literal_a",
|
@@ -148,30 +97,31 @@ class ParserTest < Test::Unit::TestCase
|
|
148
97
|
end
|
149
98
|
|
150
99
|
def test_with_a_grammar_that_is_not_SLR
|
151
|
-
grammar
|
152
|
-
parser
|
100
|
+
grammar = LALRButNotSLRGrammar
|
101
|
+
parser = Dhaka::Parser.new(grammar)
|
153
102
|
parser_input = ['b','d','c', Dhaka::END_SYMBOL_NAME]
|
154
103
|
assert_equal(["A_d", "E_bAc", "start"], get_linearized_parse_result(parser_input, parser))
|
155
104
|
end
|
156
105
|
|
157
106
|
def test_with_another_grammar_that_is_not_SLR
|
158
|
-
grammar
|
159
|
-
parser
|
107
|
+
grammar = AnotherLALRButNotSLRGrammar
|
108
|
+
parser = Dhaka::Parser.new(grammar)
|
160
109
|
parser_input = ['*', 'id', '=', 'id', Dhaka::END_SYMBOL_NAME]
|
161
110
|
assert_equal(["identifier", "l_value", "contents", "identifier", "l_value", "assignment"], get_linearized_parse_result(parser_input, parser))
|
162
111
|
end
|
163
112
|
|
164
|
-
def
|
165
|
-
grammar = RRConflictGrammar
|
113
|
+
def test_debug_output_with_a_grammar_that_should_generate_an_RR_conflict
|
166
114
|
fake_logger = FakeLogger.new
|
167
|
-
Dhaka::
|
168
|
-
|
115
|
+
Dhaka::ParserState.any_instance.stubs(:unique_name).returns("StateXXX")
|
116
|
+
parser = Dhaka::Parser.new(RRConflictGrammar, fake_logger)
|
117
|
+
num_states = parser.send(:states).size
|
118
|
+
assert_equal(['Created StateXXX.'] * num_states, fake_logger.debugs[0...num_states])
|
119
|
+
assert_equal(1, fake_logger.errors.size)
|
120
|
+
assert(fake_logger.errors.first.match(/^Parser Conflict at State:\n(.+)\n(.+)\nExisting: (.+)\nNew: (.+)\nLookahead: (.+)$/))
|
121
|
+
assert_equal(["A ::= x y ->", "B ::= x y ->", "Reduce with xy A ::= x y", "Reduce with xy_again B ::= x y", "c"],
|
122
|
+
$~[1..5].sort)
|
169
123
|
end
|
170
124
|
|
171
|
-
def set_finder(set1, set2)
|
172
|
-
set2.inject(false) {|result, member| result ||= member == set1}
|
173
|
-
end
|
174
|
-
|
175
125
|
def write_parser(parser)
|
176
126
|
File.open('parser.dot', 'w') do |file|
|
177
127
|
file << parser.to_dot
|