dhaka 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/dhaka.rb +24 -22
- data/lib/evaluator/evaluator.rb +42 -44
- data/lib/grammar/closure_hash.rb +4 -3
- data/lib/grammar/grammar.rb +113 -110
- data/lib/grammar/grammar_symbol.rb +6 -3
- data/lib/grammar/precedence.rb +3 -2
- data/lib/grammar/production.rb +5 -6
- data/lib/parser/action.rb +16 -11
- data/lib/parser/channel.rb +22 -16
- data/lib/parser/compiled_parser.rb +28 -22
- data/lib/parser/conflict.rb +54 -0
- data/lib/parser/item.rb +19 -19
- data/lib/parser/parse_result.rb +16 -1
- data/lib/parser/parse_tree.rb +15 -9
- data/lib/parser/parser.rb +51 -80
- data/lib/parser/parser_run.rb +6 -6
- data/lib/parser/parser_state.rb +16 -18
- data/lib/parser/token.rb +6 -4
- data/lib/tokenizer/tokenizer.rb +34 -31
- data/test/all_tests.rb +4 -18
- data/test/another_lalr_but_not_slr_grammar.rb +9 -5
- data/test/{arithmetic_evaluator.rb → arithmetic/arithmetic_evaluator.rb} +1 -2
- data/test/{arithmetic_evaluator_test.rb → arithmetic/arithmetic_evaluator_test.rb} +9 -20
- data/test/arithmetic/arithmetic_grammar.rb +41 -0
- data/test/{arithmetic_grammar_test.rb → arithmetic/arithmetic_grammar_test.rb} +2 -4
- data/test/{arithmetic_test_methods.rb → arithmetic/arithmetic_test_methods.rb} +1 -3
- data/test/{arithmetic_tokenizer.rb → arithmetic/arithmetic_tokenizer.rb} +8 -10
- data/test/{arithmetic_tokenizer_test.rb → arithmetic/arithmetic_tokenizer_test.rb} +4 -2
- data/test/{arithmetic_precedence_evaluator.rb → arithmetic_precedence/arithmetic_precedence_evaluator.rb} +1 -2
- data/test/arithmetic_precedence/arithmetic_precedence_grammar.rb +24 -0
- data/test/{arithmetic_precedence_grammar_test.rb → arithmetic_precedence/arithmetic_precedence_grammar_test.rb} +2 -3
- data/test/arithmetic_precedence/arithmetic_precedence_parser_test.rb +31 -0
- data/test/{arithmetic_precedence_tokenizer.rb → arithmetic_precedence/arithmetic_precedence_tokenizer.rb} +8 -10
- data/test/brackets/bracket_grammar.rb +23 -0
- data/test/{bracket_tokenizer.rb → brackets/bracket_tokenizer.rb} +2 -4
- data/test/{brackets_test.rb → brackets/brackets_test.rb} +3 -4
- data/test/chittagong/chittagong_driver.rb +47 -0
- data/test/{chittagong_driver_test.rb → chittagong/chittagong_driver_test.rb} +66 -58
- data/test/{chittagong_evaluator.rb → chittagong/chittagong_evaluator.rb} +28 -13
- data/test/{chittagong_evaluator_test.rb → chittagong/chittagong_evaluator_test.rb} +6 -10
- data/test/chittagong/chittagong_grammar.rb +110 -0
- data/test/{chittagong_parser_test.rb → chittagong/chittagong_parser_test.rb} +5 -7
- data/test/{chittagong_test.rb → chittagong/chittagong_test.rb} +27 -36
- data/test/{chittagong_tokenizer.rb → chittagong/chittagong_tokenizer.rb} +17 -17
- data/test/{chittagong_tokenizer_test.rb → chittagong/chittagong_tokenizer_test.rb} +2 -3
- data/test/compiled_parser_test.rb +9 -42
- data/test/dhaka_test_helper.rb +17 -0
- data/test/evaluator_test.rb +18 -3
- data/test/grammar_test.rb +10 -15
- data/test/lalr_but_not_slr_grammar.rb +10 -8
- data/test/malformed_grammar.rb +2 -4
- data/test/malformed_grammar_test.rb +2 -3
- data/test/nullable_grammar.rb +11 -8
- data/test/parse_result_test.rb +44 -0
- data/test/parser_state_test.rb +36 -0
- data/test/parser_test.rb +53 -103
- data/test/precedence_grammar.rb +6 -6
- data/test/precedence_grammar_test.rb +2 -3
- data/test/rr_conflict_grammar.rb +5 -7
- data/test/simple_grammar.rb +6 -8
- data/test/sr_conflict_grammar.rb +6 -6
- metadata +30 -26
- data/test/arithmetic_grammar.rb +0 -35
- data/test/arithmetic_precedence_grammar.rb +0 -24
- data/test/arithmetic_precedence_parser_test.rb +0 -33
- data/test/bracket_grammar.rb +0 -25
- data/test/chittagong_grammar.rb +0 -104
- data/test/incomplete_arithmetic_evaluator.rb +0 -60
@@ -1,21 +1,20 @@
|
|
1
|
-
require File.dirname(__FILE__)+'
|
2
|
-
require 'chittagong_grammar'
|
3
|
-
|
1
|
+
require File.dirname(__FILE__) + '/chittagong_grammar'
|
4
2
|
|
5
3
|
class ChittagongTokenizer < Dhaka::Tokenizer
|
6
4
|
|
7
|
-
KEYWORDS
|
5
|
+
KEYWORDS = %w| print if else end while def return |
|
8
6
|
|
9
|
-
digits
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
7
|
+
digits = ('0'..'9').to_a
|
8
|
+
decimal_point = %w| . |
|
9
|
+
letters = ('a'..'z').to_a
|
10
|
+
parenths = %w| ( ) |
|
11
|
+
operators = %w| - + / * ^ ! > < |
|
12
|
+
equal_sign = %w| = |
|
13
|
+
whitespace = [' ']
|
14
|
+
arg_separator = %w| , |
|
15
|
+
newline = ["\n"]
|
17
16
|
|
18
|
-
all_characters = digits + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
|
17
|
+
all_characters = digits + decimal_point + letters + parenths + operators + whitespace + newline + arg_separator + equal_sign
|
19
18
|
|
20
19
|
for_state Dhaka::TOKENIZER_IDLE_STATE do
|
21
20
|
for_characters(all_characters - (digits + letters + newline + whitespace + equal_sign)) do
|
@@ -49,7 +48,7 @@ class ChittagongTokenizer < Dhaka::Tokenizer
|
|
49
48
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
50
49
|
end
|
51
50
|
for_character equal_sign do
|
52
|
-
curr_token.symbol_name
|
51
|
+
curr_token.symbol_name << '='
|
53
52
|
advance
|
54
53
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
55
54
|
end
|
@@ -60,18 +59,18 @@ class ChittagongTokenizer < Dhaka::Tokenizer
|
|
60
59
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
61
60
|
end
|
62
61
|
for_characters digits do
|
63
|
-
curr_token.value
|
62
|
+
curr_token.value << curr_char
|
64
63
|
advance
|
65
64
|
end
|
66
65
|
end
|
67
|
-
|
66
|
+
# lipi:keywords
|
68
67
|
for_state :get_word_literal do
|
69
68
|
for_characters all_characters - letters do
|
70
69
|
curr_token.symbol_name = word_literal_symbol(curr_token.value)
|
71
70
|
switch_to Dhaka::TOKENIZER_IDLE_STATE
|
72
71
|
end
|
73
72
|
for_characters letters do
|
74
|
-
curr_token.value
|
73
|
+
curr_token.value << curr_char
|
75
74
|
advance
|
76
75
|
curr_token.symbol_name = word_literal_symbol(curr_token.value) unless curr_char
|
77
76
|
end
|
@@ -84,5 +83,6 @@ class ChittagongTokenizer < Dhaka::Tokenizer
|
|
84
83
|
'word_literal'
|
85
84
|
end
|
86
85
|
end
|
86
|
+
# lipi:keywords
|
87
87
|
|
88
88
|
end
|
@@ -1,16 +1,10 @@
|
|
1
|
-
require
|
2
|
-
require "simple_grammar"
|
3
|
-
|
4
|
-
require 'arithmetic_grammar'
|
5
|
-
|
6
|
-
eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as('Foo'))
|
7
|
-
eval(Dhaka::Parser.new(ArithmeticGrammar).compile_to_ruby_source_as('CompiledArithmeticParser'))
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + "/simple_grammar"
|
3
|
+
eval(Dhaka::Parser.new(SimpleGrammar).compile_to_ruby_source_as(:SimpleParser))
|
8
4
|
|
9
5
|
class TestCompiledParser < Test::Unit::TestCase
|
10
|
-
include ArithmeticTestMethods
|
11
|
-
|
12
6
|
def test_compiled_parser_generates_parse_tree_for_simple_grammar
|
13
|
-
parse_tree =
|
7
|
+
parse_tree = SimpleParser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#',Dhaka::END_SYMBOL_NAME]))
|
14
8
|
assert_equal \
|
15
9
|
["literal",
|
16
10
|
"term",
|
@@ -28,39 +22,13 @@ class TestCompiledParser < Test::Unit::TestCase
|
|
28
22
|
"start"], parse_tree.linearize.collect {|node| node.production.name}
|
29
23
|
end
|
30
24
|
|
31
|
-
def
|
32
|
-
|
33
|
-
|
34
|
-
["getting_literals",
|
35
|
-
"factor",
|
36
|
-
"term",
|
37
|
-
"getting_literals",
|
38
|
-
"factor",
|
39
|
-
"getting_literals",
|
40
|
-
"division",
|
41
|
-
"term",
|
42
|
-
"getting_literals",
|
43
|
-
"factor",
|
44
|
-
"subtraction",
|
45
|
-
"unpacking_parenthetized_expression",
|
46
|
-
"factor",
|
47
|
-
"getting_literals",
|
48
|
-
"division",
|
49
|
-
"subtraction",
|
50
|
-
"unpacking_parenthetized_expression",
|
51
|
-
"factor",
|
52
|
-
"term",
|
53
|
-
"expression"], parse(build_tokens(parser_input)).linearize.collect {|node| node.production.name}
|
54
|
-
end
|
55
|
-
|
56
|
-
def test_parse_result_has_nil_parse_tree_if_empty_token_array
|
57
|
-
parse_result = CompiledArithmeticParser.parse([])
|
58
|
-
assert !CompiledArithmeticParser.parse([]).has_error?
|
59
|
-
assert_nil parse_result.parse_tree
|
25
|
+
def test_parse_result_has_error_result_if_only_end_token_passed
|
26
|
+
parse_result = SimpleParser.parse(build_tokens([Dhaka::END_SYMBOL_NAME]))
|
27
|
+
assert parse_result.has_error?
|
60
28
|
end
|
61
29
|
|
62
30
|
def test_parser_returns_error_result_with_index_of_bad_token_if_parse_error
|
63
|
-
parse_result =
|
31
|
+
parse_result = SimpleParser.parse(build_tokens(['(', '-', ')',Dhaka::END_SYMBOL_NAME]))
|
64
32
|
assert parse_result.has_error?
|
65
33
|
assert_equal '-', parse_result.unexpected_token.symbol_name
|
66
34
|
end
|
@@ -68,5 +36,4 @@ class TestCompiledParser < Test::Unit::TestCase
|
|
68
36
|
def build_tokens(token_symbol_names)
|
69
37
|
token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
|
70
38
|
end
|
71
|
-
|
72
|
-
end
|
39
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
unless defined? Dhaka
|
2
|
+
require File.dirname(__FILE__) + '/../lib/dhaka'
|
3
|
+
end
|
4
|
+
|
5
|
+
require File.dirname(__FILE__) + '/fake_logger'
|
6
|
+
|
7
|
+
require 'test/unit'
|
8
|
+
|
9
|
+
begin
|
10
|
+
require 'mocha'
|
11
|
+
rescue LoadError
|
12
|
+
puts "
|
13
|
+
The tests depend on Mocha. Please install it if you wish to run them:
|
14
|
+
sudo gem install mocha
|
15
|
+
This gem is not required for using the library."
|
16
|
+
exit
|
17
|
+
end
|
data/test/evaluator_test.rb
CHANGED
@@ -1,8 +1,23 @@
|
|
1
|
-
require
|
2
|
-
require File.dirname(__FILE__)+'
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
3
3
|
|
4
4
|
class TestEvaluator < Test::Unit::TestCase
|
5
5
|
def test_throws_exception_if_evaluation_rules_not_completely_defined
|
6
|
-
assert_raise(Dhaka::EvaluatorDefinitionError)
|
6
|
+
assert_raise(Dhaka::EvaluatorDefinitionError) do
|
7
|
+
eval(
|
8
|
+
"class IncompleteSimpleEvaluator < Dhaka::Evaluator
|
9
|
+
self.grammar = SimpleGrammar
|
10
|
+
define_evaluation_rules do
|
11
|
+
for_start do
|
12
|
+
something
|
13
|
+
end
|
14
|
+
|
15
|
+
for_literal do
|
16
|
+
something
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end")
|
20
|
+
end
|
21
|
+
|
7
22
|
end
|
8
23
|
end
|
data/test/grammar_test.rb
CHANGED
@@ -1,35 +1,32 @@
|
|
1
|
-
|
2
|
-
require '
|
3
|
-
require 'simple_grammar'
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
4
3
|
|
5
4
|
class SimpleGrammarTest < Test::Unit::TestCase
|
6
|
-
|
7
5
|
def setup
|
8
6
|
@grammar = SimpleGrammar
|
9
7
|
end
|
10
8
|
|
11
9
|
def test_loads_symbol_and_classifies_them
|
12
10
|
expected_non_terminals = Set.new(['E', 'S', 'T', Dhaka::START_SYMBOL_NAME])
|
13
|
-
expected_terminals
|
11
|
+
expected_terminals = Set.new(['-', 'n', '(', ')', '#', Dhaka::END_SYMBOL_NAME])
|
14
12
|
assert_equal(expected_non_terminals, Set.new(@grammar.non_terminal_symbols.collect {|symbol| symbol.name}))
|
15
13
|
assert_equal(expected_terminals, Set.new(@grammar.terminal_symbols.collect {|symbol| symbol.name}))
|
16
14
|
end
|
17
15
|
|
18
16
|
def test_creates_productions
|
19
17
|
productions_for_E = @grammar.productions_for_symbol(@grammar.symbol_for_name('E'))
|
20
|
-
assert(productions_for_E.size==2)
|
18
|
+
assert(productions_for_E.size == 2)
|
21
19
|
expected_productions_for_E = Set.new(['subtraction E ::= E - T', 'term E ::= T'])
|
22
20
|
assert_equal(expected_productions_for_E, Set.new(productions_for_E.collect {|production| production.to_s}))
|
23
21
|
productions_for_start = @grammar.productions_for_symbol(@grammar.start_symbol)
|
24
|
-
assert(productions_for_start.size==1)
|
22
|
+
assert(productions_for_start.size == 1)
|
25
23
|
expected_productions_for_start = Set.new(['start _Start_ ::= S #'])
|
26
24
|
assert_equal(expected_productions_for_start, Set.new(productions_for_start.collect {|production| production.to_s}))
|
27
|
-
|
28
25
|
end
|
29
26
|
|
30
27
|
def test_symbols_in_productions_use_the_flyweight_pattern
|
31
28
|
assert_same(@grammar.production_named('subtraction').symbol, @grammar.production_named('term').symbol)
|
32
|
-
assert_same(@grammar.production_named('expression').expansion
|
29
|
+
assert_same(@grammar.production_named('expression').expansion.first, @grammar.production_named('subtraction').expansion.first)
|
33
30
|
end
|
34
31
|
|
35
32
|
def test_first_with_non_terminal
|
@@ -43,11 +40,11 @@ class SimpleGrammarTest < Test::Unit::TestCase
|
|
43
40
|
end
|
44
41
|
|
45
42
|
def test_computes_closures_and_channels_given_a_kernel
|
46
|
-
start_production
|
47
|
-
start_item
|
48
|
-
kernel
|
43
|
+
start_production = @grammar.production_named('start')
|
44
|
+
start_item = Dhaka::Item.new(start_production, 0)
|
45
|
+
kernel = Set.new([start_item])
|
49
46
|
channels, closure = @grammar.closure(kernel)
|
50
|
-
expected_items
|
47
|
+
expected_items = Set.new(['_Start_ ::= -> S # []',
|
51
48
|
'S ::= -> E []',
|
52
49
|
'E ::= -> E - T []',
|
53
50
|
'E ::= -> T []',
|
@@ -65,6 +62,4 @@ class SimpleGrammarTest < Test::Unit::TestCase
|
|
65
62
|
assert_equal(expected_items, Set.new(closure.values.collect{|item| item.to_s}))
|
66
63
|
assert_equal(expected_channels, Set.new(channels.collect{|item| item.to_s}))
|
67
64
|
end
|
68
|
-
|
69
|
-
|
70
65
|
end
|
@@ -1,17 +1,19 @@
|
|
1
|
-
require File.dirname(__FILE__)+'/../lib/dhaka'
|
2
|
-
|
3
1
|
class LALRButNotSLRGrammar < Dhaka::Grammar
|
2
|
+
|
4
3
|
for_symbol(Dhaka::START_SYMBOL_NAME) do
|
5
|
-
start
|
4
|
+
start %w| E |
|
6
5
|
end
|
6
|
+
|
7
7
|
for_symbol('E') do
|
8
|
-
E_Aa
|
9
|
-
E_bAc
|
10
|
-
E_dc
|
11
|
-
E_bda
|
8
|
+
E_Aa %w| A a |
|
9
|
+
E_bAc %w| b A c |
|
10
|
+
E_dc %w| d c |
|
11
|
+
E_bda %w| b d a |
|
12
12
|
end
|
13
|
+
|
13
14
|
for_symbol('A') do
|
14
|
-
A_d
|
15
|
+
A_d %w| d |
|
15
16
|
end
|
17
|
+
|
16
18
|
end
|
17
19
|
|
data/test/malformed_grammar.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
|
-
require
|
2
|
-
|
3
|
-
require "malformed_grammar"
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + "/malformed_grammar"
|
4
3
|
|
5
4
|
class TestMalformedGrammar < Test::Unit::TestCase
|
6
5
|
def test_must_have_a_start_symbol_in_order_to_generate_a_parser
|
data/test/nullable_grammar.rb
CHANGED
@@ -1,18 +1,21 @@
|
|
1
|
-
require File.dirname(__FILE__)+'/../lib/dhaka'
|
2
|
-
|
3
1
|
class NullableGrammar < Dhaka::Grammar
|
2
|
+
|
4
3
|
for_symbol(Dhaka::START_SYMBOL_NAME) do
|
5
|
-
tuple
|
4
|
+
tuple %w| Tuple |
|
6
5
|
end
|
6
|
+
|
7
7
|
for_symbol('Tuple') do
|
8
|
-
element_list
|
8
|
+
element_list %w| ( Elements ) |
|
9
9
|
end
|
10
|
+
|
10
11
|
for_symbol('Elements') do
|
11
|
-
empty_element_list
|
12
|
-
concatenate_element_lists
|
12
|
+
empty_element_list %w| |
|
13
|
+
concatenate_element_lists %w| Character Elements |
|
13
14
|
end
|
15
|
+
|
14
16
|
for_symbol('Character') do
|
15
|
-
literal_a
|
16
|
-
literal_b
|
17
|
+
literal_a %w| a |
|
18
|
+
literal_b %w| b |
|
17
19
|
end
|
20
|
+
|
18
21
|
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
3
|
+
|
4
|
+
class TestParseSuccessResult < Test::Unit::TestCase
|
5
|
+
include Dhaka
|
6
|
+
|
7
|
+
def composite_node(production, child_nodes, dot_name)
|
8
|
+
node = ParseTreeCompositeNode.new(SimpleGrammar.production_named(production))
|
9
|
+
node.child_nodes.concat child_nodes
|
10
|
+
node.stubs(:dot_name).returns(dot_name)
|
11
|
+
node
|
12
|
+
end
|
13
|
+
|
14
|
+
def leaf_node(token, value, dot_name)
|
15
|
+
node = ParseTreeLeafNode.new(Token.new(token, value, nil))
|
16
|
+
node.stubs(:dot_name).returns(dot_name)
|
17
|
+
node
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_parse_tree_can_be_exported_to_dot_format
|
21
|
+
first_term = composite_node('literal', [leaf_node('n', 1, "literal_1")], "first_term")
|
22
|
+
second_term = composite_node('literal', [leaf_node('n', 2, "literal_2")], "second_term")
|
23
|
+
addition_term = leaf_node('-', nil, "subtraction_operator")
|
24
|
+
tree = composite_node('subtraction', [first_term, addition_term, second_term], "expression")
|
25
|
+
result = ParseSuccessResult.new(tree)
|
26
|
+
assert_equal(
|
27
|
+
%(digraph x {
|
28
|
+
node [fontsize="10" shape=box size="5"]
|
29
|
+
expression [label="subtraction E ::= E - T"]
|
30
|
+
expression -> first_term
|
31
|
+
first_term [label="literal T ::= n"]
|
32
|
+
first_term -> literal_1
|
33
|
+
literal_1 [label="n : 1"]
|
34
|
+
expression -> subtraction_operator
|
35
|
+
subtraction_operator [label="-"]
|
36
|
+
expression -> second_term
|
37
|
+
second_term [label="literal T ::= n"]
|
38
|
+
second_term -> literal_2
|
39
|
+
literal_2 [label="n : 2"]
|
40
|
+
}),
|
41
|
+
result.to_dot)
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
|
3
|
+
class TestParserState < Test::Unit::TestCase
|
4
|
+
include Dhaka
|
5
|
+
def test_unique_identifier
|
6
|
+
states = (0...5).collect {|i| ParserState.new(nil, {})}
|
7
|
+
ids = Set.new
|
8
|
+
states.each do |state|
|
9
|
+
assert(/^State(\d+)$/ =~ state.unique_name)
|
10
|
+
ids << $1.to_i
|
11
|
+
end
|
12
|
+
assert_equal(5, ids.size)
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_to_s_method
|
16
|
+
options = {:some_option => true}
|
17
|
+
item1 = mock()
|
18
|
+
item1.stubs(:to_s).with(options).returns("i'm item 1")
|
19
|
+
item2 = mock()
|
20
|
+
item2.stubs(:to_s).with(options).returns("i'm item 2")
|
21
|
+
state = ParserState.new(nil, {item1 => item1, item2 => item2})
|
22
|
+
assert_equal(["i'm item 1", "i'm item 2"], state.to_s(options).split("\n").sort)
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_to_dot_method
|
26
|
+
options = {:some_option => true}
|
27
|
+
item1 = mock()
|
28
|
+
item1.stubs(:to_s).with(options).returns("i'm item 1")
|
29
|
+
item2 = mock()
|
30
|
+
item2.stubs(:to_s).with(options).returns("i'm item 2")
|
31
|
+
state = ParserState.new(nil, {item1 => item1, item2 => item2})
|
32
|
+
state.stubs(:unique_name).returns("State1")
|
33
|
+
assert(state.to_dot(options) =~ /^State1 \[label="(.+)\\n(.+)"\]/)
|
34
|
+
assert_equal(["i'm item 1", "i'm item 2"], [$1, $2].sort)
|
35
|
+
end
|
36
|
+
end
|
data/test/parser_test.rb
CHANGED
@@ -1,16 +1,12 @@
|
|
1
|
-
|
2
|
-
require '
|
3
|
-
require '
|
4
|
-
require '
|
5
|
-
require '
|
6
|
-
require '
|
7
|
-
require '
|
8
|
-
require 'rr_conflict_grammar'
|
9
|
-
require 'sr_conflict_grammar'
|
10
|
-
require 'fake_logger'
|
1
|
+
require File.dirname(__FILE__) + '/dhaka_test_helper'
|
2
|
+
require File.dirname(__FILE__) + '/simple_grammar'
|
3
|
+
require File.dirname(__FILE__) + '/nullable_grammar'
|
4
|
+
require File.dirname(__FILE__) + '/lalr_but_not_slr_grammar'
|
5
|
+
require File.dirname(__FILE__) + '/another_lalr_but_not_slr_grammar'
|
6
|
+
require File.dirname(__FILE__) + '/rr_conflict_grammar'
|
7
|
+
require File.dirname(__FILE__) + '/sr_conflict_grammar'
|
11
8
|
|
12
9
|
class ParserTest < Test::Unit::TestCase
|
13
|
-
|
14
10
|
def build_tokens(token_symbol_names)
|
15
11
|
token_symbol_names.collect {|symbol_name| Dhaka::Token.new(symbol_name, nil, nil)}
|
16
12
|
end
|
@@ -20,37 +16,45 @@ class ParserTest < Test::Unit::TestCase
|
|
20
16
|
@parser = Dhaka::Parser.new(@grammar)
|
21
17
|
end
|
22
18
|
|
19
|
+
def contains(given_member, set)
|
20
|
+
set.inject(false) {|result, member| result ||= (Set.new(member) == Set.new(given_member))}
|
21
|
+
end
|
22
|
+
|
23
|
+
def assert_collection_equal(expected, actual)
|
24
|
+
assert_equal(expected.size, actual.size)
|
25
|
+
actual.each do |actual_member|
|
26
|
+
assert(contains(actual_member, expected))
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
23
30
|
def test_parser_generates_states_with_correct_items
|
24
|
-
expected_states =
|
25
|
-
|
31
|
+
expected_states = [
|
32
|
+
['_Start_ ::= -> S # [_End_]',
|
26
33
|
'S ::= -> E [#]',
|
27
34
|
'E ::= -> E - T [#-]',
|
28
35
|
'E ::= -> T [#-]',
|
29
36
|
'T ::= -> n [#-]',
|
30
|
-
'T ::= -> ( E ) [#-]']
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
'E ::= E -> - T [#-]']
|
35
|
-
|
36
|
-
|
37
|
+
'T ::= -> ( E ) [#-]'],
|
38
|
+
['E ::= T -> [#)-]'],
|
39
|
+
['T ::= n -> [#)-]'],
|
40
|
+
['S ::= E -> [#]',
|
41
|
+
'E ::= E -> - T [#-]'],
|
42
|
+
['_Start_ ::= S -> # [_End_]'],
|
43
|
+
['T ::= ( -> E ) [#)-]',
|
37
44
|
'E ::= -> E - T [)-]',
|
38
45
|
'E ::= -> T [)-]',
|
39
46
|
'T ::= -> n [)-]',
|
40
|
-
'T ::= -> ( E ) [)-]']
|
41
|
-
|
47
|
+
'T ::= -> ( E ) [)-]'],
|
48
|
+
['E ::= E - -> T [#)-]',
|
42
49
|
'T ::= -> n [#)-]',
|
43
|
-
'T ::= -> ( E ) [#)-]']
|
44
|
-
|
45
|
-
|
46
|
-
'E ::= E -> - T [)-]']
|
47
|
-
|
48
|
-
|
49
|
-
actual_states =
|
50
|
-
|
51
|
-
expected_states.values.each do |state|
|
52
|
-
assert set_finder(state, actual_states), "expected #{state.to_a}"
|
53
|
-
end
|
50
|
+
'T ::= -> ( E ) [#)-]'],
|
51
|
+
['E ::= E - T -> [#)-]'],
|
52
|
+
['T ::= ( E -> ) [#)-]',
|
53
|
+
'E ::= E -> - T [)-]'],
|
54
|
+
['T ::= ( E ) -> [#)-]'],
|
55
|
+
['_Start_ ::= S # -> [_End_]']]
|
56
|
+
actual_states = @parser.send('states').collect {|state| state.items.values.collect {|item| item.to_s}}
|
57
|
+
assert_collection_equal(expected_states, actual_states)
|
54
58
|
end
|
55
59
|
|
56
60
|
def test_parser_can_be_exported_to_dot_format
|
@@ -58,7 +62,7 @@ class ParserTest < Test::Unit::TestCase
|
|
58
62
|
end
|
59
63
|
|
60
64
|
def test_parser_generates_parse_tree_given_a_stream_of_symbols
|
61
|
-
parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME]))
|
65
|
+
parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME]))
|
62
66
|
assert_equal \
|
63
67
|
["literal",
|
64
68
|
"term",
|
@@ -76,68 +80,13 @@ class ParserTest < Test::Unit::TestCase
|
|
76
80
|
"start"], parse_tree.linearize.collect {|node| node.production.name}
|
77
81
|
end
|
78
82
|
|
79
|
-
def test_parse_trees_can_be_exported_to_dot_format
|
80
|
-
parse_tree = @parser.parse(build_tokens(['(','n','-','(','n','-','n',')',')','-','n','#', Dhaka::END_SYMBOL_NAME])).parse_tree
|
81
|
-
parse_tree.to_dot
|
82
|
-
end
|
83
|
-
|
84
83
|
def get_linearized_parse_result(input, parser)
|
85
84
|
parser.parse(build_tokens(input)).parse_tree.linearize.collect {|node| node.production.name}
|
86
85
|
end
|
87
86
|
|
88
|
-
def test_with_a_different_grammar_with_division
|
89
|
-
grammar = ArithmeticGrammar
|
90
|
-
parser = Dhaka::Parser.new(grammar)
|
91
|
-
parser_input = ['(','n','-','(','n','/','n','-','n',')','/','n',')', Dhaka::END_SYMBOL_NAME]
|
92
|
-
assert_equal \
|
93
|
-
["getting_literals",
|
94
|
-
"factor",
|
95
|
-
"term",
|
96
|
-
"getting_literals",
|
97
|
-
"factor",
|
98
|
-
"getting_literals",
|
99
|
-
"division",
|
100
|
-
"term",
|
101
|
-
"getting_literals",
|
102
|
-
"factor",
|
103
|
-
"subtraction",
|
104
|
-
"unpacking_parenthetized_expression",
|
105
|
-
"factor",
|
106
|
-
"getting_literals",
|
107
|
-
"division",
|
108
|
-
"subtraction",
|
109
|
-
"unpacking_parenthetized_expression",
|
110
|
-
"factor",
|
111
|
-
"term",
|
112
|
-
"expression"], get_linearized_parse_result(parser_input, parser)
|
113
|
-
|
114
|
-
parser_input = ['h','(','(','n',')','-','n',',','n',')', Dhaka::END_SYMBOL_NAME]
|
115
|
-
assert_equal \
|
116
|
-
["max_function",
|
117
|
-
"getting_literals",
|
118
|
-
"factor",
|
119
|
-
"term",
|
120
|
-
"unpacking_parenthetized_expression",
|
121
|
-
"factor",
|
122
|
-
"term",
|
123
|
-
"getting_literals",
|
124
|
-
"factor",
|
125
|
-
"subtraction",
|
126
|
-
"getting_literals",
|
127
|
-
"factor",
|
128
|
-
"term",
|
129
|
-
"single_args",
|
130
|
-
"concatenating_args",
|
131
|
-
"evaluating_function",
|
132
|
-
"function",
|
133
|
-
"factor",
|
134
|
-
"term",
|
135
|
-
"expression"], get_linearized_parse_result(parser_input, parser)
|
136
|
-
end
|
137
|
-
|
138
87
|
def test_with_a_grammar_with_nullables_after_terminals
|
139
|
-
grammar
|
140
|
-
parser
|
88
|
+
grammar = NullableGrammar
|
89
|
+
parser = Dhaka::Parser.new(grammar)
|
141
90
|
parser_input = ['(','a',')', Dhaka::END_SYMBOL_NAME]
|
142
91
|
assert_equal \
|
143
92
|
["literal_a",
|
@@ -148,30 +97,31 @@ class ParserTest < Test::Unit::TestCase
|
|
148
97
|
end
|
149
98
|
|
150
99
|
def test_with_a_grammar_that_is_not_SLR
|
151
|
-
grammar
|
152
|
-
parser
|
100
|
+
grammar = LALRButNotSLRGrammar
|
101
|
+
parser = Dhaka::Parser.new(grammar)
|
153
102
|
parser_input = ['b','d','c', Dhaka::END_SYMBOL_NAME]
|
154
103
|
assert_equal(["A_d", "E_bAc", "start"], get_linearized_parse_result(parser_input, parser))
|
155
104
|
end
|
156
105
|
|
157
106
|
def test_with_another_grammar_that_is_not_SLR
|
158
|
-
grammar
|
159
|
-
parser
|
107
|
+
grammar = AnotherLALRButNotSLRGrammar
|
108
|
+
parser = Dhaka::Parser.new(grammar)
|
160
109
|
parser_input = ['*', 'id', '=', 'id', Dhaka::END_SYMBOL_NAME]
|
161
110
|
assert_equal(["identifier", "l_value", "contents", "identifier", "l_value", "assignment"], get_linearized_parse_result(parser_input, parser))
|
162
111
|
end
|
163
112
|
|
164
|
-
def
|
165
|
-
grammar = RRConflictGrammar
|
113
|
+
def test_debug_output_with_a_grammar_that_should_generate_an_RR_conflict
|
166
114
|
fake_logger = FakeLogger.new
|
167
|
-
Dhaka::
|
168
|
-
|
115
|
+
Dhaka::ParserState.any_instance.stubs(:unique_name).returns("StateXXX")
|
116
|
+
parser = Dhaka::Parser.new(RRConflictGrammar, fake_logger)
|
117
|
+
num_states = parser.send(:states).size
|
118
|
+
assert_equal(['Created StateXXX.'] * num_states, fake_logger.debugs[0...num_states])
|
119
|
+
assert_equal(1, fake_logger.errors.size)
|
120
|
+
assert(fake_logger.errors.first.match(/^Parser Conflict at State:\n(.+)\n(.+)\nExisting: (.+)\nNew: (.+)\nLookahead: (.+)$/))
|
121
|
+
assert_equal(["A ::= x y ->", "B ::= x y ->", "Reduce with xy A ::= x y", "Reduce with xy_again B ::= x y", "c"],
|
122
|
+
$~[1..5].sort)
|
169
123
|
end
|
170
124
|
|
171
|
-
def set_finder(set1, set2)
|
172
|
-
set2.inject(false) {|result, member| result ||= member == set1}
|
173
|
-
end
|
174
|
-
|
175
125
|
def write_parser(parser)
|
176
126
|
File.open('parser.dot', 'w') do |file|
|
177
127
|
file << parser.to_dot
|