antlr3 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/ANTLR-LICENSE.txt +26 -0
  2. data/History.txt +66 -0
  3. data/README.txt +139 -0
  4. data/bin/antlr4ruby +33 -0
  5. data/java/RubyTarget.java +524 -0
  6. data/java/antlr-full-3.2.1.jar +0 -0
  7. data/lib/antlr3.rb +176 -0
  8. data/lib/antlr3/constants.rb +88 -0
  9. data/lib/antlr3/debug.rb +701 -0
  10. data/lib/antlr3/debug/event-hub.rb +210 -0
  11. data/lib/antlr3/debug/record-event-listener.rb +25 -0
  12. data/lib/antlr3/debug/rule-tracer.rb +55 -0
  13. data/lib/antlr3/debug/socket.rb +360 -0
  14. data/lib/antlr3/debug/trace-event-listener.rb +92 -0
  15. data/lib/antlr3/dfa.rb +247 -0
  16. data/lib/antlr3/dot.rb +174 -0
  17. data/lib/antlr3/error.rb +657 -0
  18. data/lib/antlr3/main.rb +561 -0
  19. data/lib/antlr3/modes/ast-builder.rb +41 -0
  20. data/lib/antlr3/modes/filter.rb +56 -0
  21. data/lib/antlr3/profile.rb +322 -0
  22. data/lib/antlr3/recognizers.rb +1280 -0
  23. data/lib/antlr3/streams.rb +985 -0
  24. data/lib/antlr3/streams/interactive.rb +91 -0
  25. data/lib/antlr3/streams/rewrite.rb +412 -0
  26. data/lib/antlr3/test/call-stack.rb +57 -0
  27. data/lib/antlr3/test/config.rb +23 -0
  28. data/lib/antlr3/test/core-extensions.rb +269 -0
  29. data/lib/antlr3/test/diff.rb +165 -0
  30. data/lib/antlr3/test/functional.rb +207 -0
  31. data/lib/antlr3/test/grammar.rb +371 -0
  32. data/lib/antlr3/token.rb +592 -0
  33. data/lib/antlr3/tree.rb +1415 -0
  34. data/lib/antlr3/tree/debug.rb +163 -0
  35. data/lib/antlr3/tree/visitor.rb +84 -0
  36. data/lib/antlr3/tree/wizard.rb +481 -0
  37. data/lib/antlr3/util.rb +149 -0
  38. data/lib/antlr3/version.rb +27 -0
  39. data/samples/ANTLRv3Grammar.g +621 -0
  40. data/samples/Cpp.g +749 -0
  41. data/templates/AST.stg +335 -0
  42. data/templates/ASTDbg.stg +40 -0
  43. data/templates/ASTParser.stg +153 -0
  44. data/templates/ASTTreeParser.stg +272 -0
  45. data/templates/Dbg.stg +192 -0
  46. data/templates/Ruby.stg +1514 -0
  47. data/test/functional/ast-output/auto-ast.rb +797 -0
  48. data/test/functional/ast-output/construction.rb +555 -0
  49. data/test/functional/ast-output/hetero-nodes.rb +753 -0
  50. data/test/functional/ast-output/rewrites.rb +1327 -0
  51. data/test/functional/ast-output/tree-rewrite.rb +1662 -0
  52. data/test/functional/debugging/debug-mode.rb +689 -0
  53. data/test/functional/debugging/profile-mode.rb +165 -0
  54. data/test/functional/debugging/rule-tracing.rb +74 -0
  55. data/test/functional/delegation/import.rb +379 -0
  56. data/test/functional/lexer/basic.rb +559 -0
  57. data/test/functional/lexer/filter-mode.rb +245 -0
  58. data/test/functional/lexer/nuances.rb +47 -0
  59. data/test/functional/lexer/properties.rb +104 -0
  60. data/test/functional/lexer/syn-pred.rb +32 -0
  61. data/test/functional/lexer/xml.rb +206 -0
  62. data/test/functional/main/main-scripts.rb +245 -0
  63. data/test/functional/parser/actions.rb +224 -0
  64. data/test/functional/parser/backtracking.rb +244 -0
  65. data/test/functional/parser/basic.rb +282 -0
  66. data/test/functional/parser/calc.rb +98 -0
  67. data/test/functional/parser/ll-star.rb +143 -0
  68. data/test/functional/parser/nuances.rb +165 -0
  69. data/test/functional/parser/predicates.rb +103 -0
  70. data/test/functional/parser/properties.rb +242 -0
  71. data/test/functional/parser/rule-methods.rb +132 -0
  72. data/test/functional/parser/scopes.rb +274 -0
  73. data/test/functional/token-rewrite/basic.rb +318 -0
  74. data/test/functional/token-rewrite/via-parser.rb +100 -0
  75. data/test/functional/tree-parser/basic.rb +750 -0
  76. data/test/unit/sample-input/file-stream-1 +2 -0
  77. data/test/unit/sample-input/teststreams.input2 +2 -0
  78. data/test/unit/test-dfa.rb +52 -0
  79. data/test/unit/test-exceptions.rb +44 -0
  80. data/test/unit/test-recognizers.rb +55 -0
  81. data/test/unit/test-scheme.rb +62 -0
  82. data/test/unit/test-streams.rb +459 -0
  83. data/test/unit/test-tree-wizard.rb +535 -0
  84. data/test/unit/test-trees.rb +854 -0
  85. metadata +205 -0
@@ -0,0 +1,318 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestRewritingLexerOutputDirectly < ANTLR3::Test::Functional
7
+ inline_grammar(<<-'END')
8
+ lexer grammar SimpleRewriting;
9
+ options {
10
+ language = Ruby;
11
+ }
12
+
13
+ A: 'a';
14
+ B: 'b';
15
+ C: 'c';
16
+ END
17
+
18
+ def rewrite(input, expected)
19
+ lexer = SimpleRewriting::Lexer.new(input)
20
+ tokens = ANTLR3::TokenRewriteStream.new(lexer)
21
+ yield(tokens)
22
+ tokens.render.should == expected
23
+ return(tokens)
24
+ end
25
+
26
+ example 'insert before' do
27
+ rewrite( 'abc', '0abc' ) do |stream|
28
+ stream.insert_before(0, '0')
29
+ end
30
+ end
31
+
32
+ example 'insert after last index' do
33
+ rewrite( 'abc', 'abcx' ) do |stream|
34
+ stream.insert_after(2, 'x')
35
+ end
36
+ end
37
+
38
+ example 'insert before after middle index' do
39
+ rewrite( 'abc', 'axbxc' ) do |stream|
40
+ stream.insert_before 1, 'x'
41
+ stream.insert_after 1, 'x'
42
+ end
43
+ end
44
+
45
+ example 'replace index 0' do
46
+ rewrite( 'abc', 'xbc' ) do |stream|
47
+ stream.replace(0, 'x')
48
+ end
49
+ end
50
+
51
+ example 'replace last index' do
52
+ rewrite( 'abc', 'abx' ) do |stream|
53
+ stream.replace 2, 'x'
54
+ end
55
+ end
56
+
57
+ example 'replace last index' do
58
+ rewrite( 'abc', 'abx' ) do |stream|
59
+ stream.replace(2, 'x')
60
+ end
61
+ end
62
+
63
+ example 'replace middle index' do
64
+ rewrite( 'abc', 'axc' ) do |stream|
65
+ stream.replace 1, 'x'
66
+ end
67
+ end
68
+
69
+ example 'replace middle index' do
70
+ rewrite( 'abc', 'ayc' ) do |stream|
71
+ stream.replace 1, 'x'
72
+ stream.replace 1, 'y'
73
+ end
74
+ end
75
+
76
+ example 'replace middle index 1 insert before' do
77
+ rewrite( 'abc', '_ayc' ) do |stream|
78
+ stream.insert_before 0, '_'
79
+ stream.replace 1, 'x'
80
+ stream.replace 1, 'y'
81
+ end
82
+ end
83
+
84
+ example 'replace then delete middle index' do
85
+ rewrite( 'abc', 'ac' ) do |stream|
86
+ stream.replace 1, 'x'
87
+ stream.delete 1
88
+ end
89
+ end
90
+
91
+ example 'insert then replace same index' do
92
+ rewrite 'abc', 'xbc' do |stream|
93
+ stream.insert_before 0, '0'
94
+ stream.replace 0, 'x'
95
+ end
96
+ end
97
+
98
+ example 'insert middle index' do
99
+ rewrite( "abc", "ayxbc" ) do |stream|
100
+ stream.insert_before(1, "x")
101
+ stream.insert_before(1, "y")
102
+ end
103
+ end
104
+
105
+ example 'insert then replace index0' do
106
+ rewrite( "abc", "zbc" ) do |stream|
107
+ stream.insert_before(0, "x")
108
+ stream.insert_before(0, "y")
109
+ stream.replace(0, "z")
110
+ end
111
+ end
112
+
113
+ example 'replace then insert before last index' do
114
+ rewrite( "abc", "abyx" ) do |stream|
115
+ stream.replace(2, "x")
116
+ stream.insert_before(2, "y")
117
+ end
118
+ end
119
+
120
+ example 'insert then replace last index' do
121
+ rewrite( "abc", "abx" ) do |stream|
122
+ stream.insert_before(2, "y")
123
+ stream.replace(2, "x")
124
+ end
125
+ end
126
+
127
+ example 'replace then insert after last index' do
128
+ rewrite( "abc", "abxy" ) do |stream|
129
+ stream.replace(2, "x")
130
+ stream.insert_after(2, "y")
131
+ end
132
+ end
133
+
134
+ example 'replace range then insert at left edge' do
135
+ rewrite( "abcccba", "abyxba" ) do |stream|
136
+ stream.replace(2, 4, "x")
137
+ stream.insert_before(2, "y")
138
+ end
139
+ end
140
+
141
+ example 'replace range then insert after right edge' do
142
+ rewrite( "abcccba", "abxyba" ) do |stream|
143
+ stream.replace(2, 4, "x")
144
+ stream.insert_after(4, "y")
145
+ end
146
+ end
147
+
148
+ example 'replace all' do
149
+ rewrite( "abcccba", "x" ) do |stream|
150
+ stream.replace(0, 6, "x")
151
+ end
152
+ end
153
+
154
+ example 'replace single middle then overlapping superset' do
155
+ rewrite( "abcba", "fooa" ) do |stream|
156
+ stream.replace(2, 2, "xyz")
157
+ stream.replace(0, 3, "foo")
158
+ end
159
+ end
160
+
161
+ example 'combine inserts' do
162
+ rewrite( "abc", "yxabc" ) do |stream|
163
+ stream.insert_before(0, "x")
164
+ stream.insert_before(0, "y")
165
+ end
166
+ end
167
+
168
+ example 'combine3 inserts' do
169
+ rewrite( "abc", "yazxbc" ) do |stream|
170
+ stream.insert_before(1, "x")
171
+ stream.insert_before(0, "y")
172
+ stream.insert_before(1, "z")
173
+ end
174
+ end
175
+
176
+ example 'disjoint inserts' do
177
+ rewrite( "abc", "zaxbyc" ) do |stream|
178
+ stream.insert_before(1, "x")
179
+ stream.insert_before(2, "y")
180
+ stream.insert_before(0, "z")
181
+ end
182
+ end
183
+
184
+ example 'leave alone disjoint insert' do
185
+ rewrite( "abcc", "axbfoo" ) do |stream|
186
+ stream.insert_before(1, "x")
187
+ stream.replace(2, 3, "foo")
188
+ end
189
+ end
190
+
191
+ example 'leave alone disjoint insert2' do
192
+ rewrite( "abcc", "axbfoo" ) do |stream|
193
+ stream.replace(2, 3, "foo")
194
+ stream.insert_before(1, "x")
195
+ end
196
+ end
197
+
198
+ example 'combine insert on left with delete' do
199
+ rewrite( "abc", "z" ) do |stream|
200
+ stream.delete(0, 2)
201
+ stream.insert_before(0, "z")
202
+ end
203
+ end
204
+
205
+ example 'overlapping replace' do
206
+ rewrite( "abcc", "bar" ) do |stream|
207
+ stream.replace(1, 2, "foo")
208
+ stream.replace(0, 3, "bar")
209
+ end
210
+ end
211
+
212
+ example 'overlapping replace3' do
213
+ rewrite( "abcc", "barc" ) do |stream|
214
+ stream.replace(1, 2, "foo")
215
+ stream.replace(0, 2, "bar")
216
+ end
217
+ end
218
+
219
+ example 'overlapping replace 4' do
220
+ rewrite( "abcc", "abar" ) do |stream|
221
+ stream.replace(1, 2, "foo")
222
+ stream.replace(1, 3, "bar")
223
+ end
224
+ end
225
+
226
+ example 'overlapping replace 2' do
227
+ lexer = SimpleRewriting::Lexer.new( 'abcc' )
228
+ stream = ANTLR3::TokenRewriteStream.new( lexer )
229
+ stream.replace 0, 3, 'bar'
230
+ stream.replace 1, 2, 'foo'
231
+
232
+ lambda { stream.render }.
233
+ should raise_error { |error|
234
+ error.to_s.should == %q<operation (replace @ 1..2 : "foo") overlaps with previous operation (replace @ 0..3 : "bar")>
235
+ }
236
+ end
237
+
238
+ example 'replace range then insert at right edge' do
239
+ lexer = SimpleRewriting::Lexer.new( 'abcccba' )
240
+ stream = ANTLR3::TokenRewriteStream.new( lexer )
241
+ stream.replace 2, 4, 'x'
242
+ stream.insert_before 4, 'y'
243
+ lambda { stream.render }.
244
+ should raise_error { |error|
245
+ error.to_s.should == %q<operation (insert-before @ 4 : "y") overlaps with previous operation (replace @ 2..4 : "x")>
246
+ }
247
+ end
248
+
249
+ example 'replace then replace superset' do
250
+ lexer = SimpleRewriting::Lexer.new( 'abcccba' )
251
+ stream = ANTLR3::TokenRewriteStream.new( lexer )
252
+ stream.replace 2, 4, 'xyz'
253
+ stream.replace 3, 5, 'foo'
254
+ lambda { stream.render }.
255
+ should raise_error { |error|
256
+ error.to_s.should == %q<operation (replace @ 3..5 : "foo") overlaps with previous operation (replace @ 2..4 : "xyz")>
257
+ }
258
+ end
259
+
260
+ example 'replace then replace lower indexed superset' do
261
+ lexer = SimpleRewriting::Lexer.new( 'abcccba' )
262
+ stream = ANTLR3::TokenRewriteStream.new( lexer )
263
+ stream.replace 2, 4, 'xyz'
264
+ stream.replace 1, 3, 'foo'
265
+ lambda { stream.render }.
266
+ should raise_error { |error|
267
+ error.to_s.should == %q<operation (replace @ 1..3 : "foo") overlaps with previous operation (replace @ 2..4 : "xyz")>
268
+ }
269
+ end
270
+
271
+ end
272
+
273
+ class TestRewritingWithTokenStream2 < ANTLR3::Test::Functional
274
+ inline_grammar( <<-END )
275
+ lexer grammar SimpleRewriting2;
276
+ options {
277
+ language = Ruby;
278
+ }
279
+
280
+ ID : 'a'..'z'+;
281
+ INT : '0'..'9'+;
282
+ SEMI : ';';
283
+ PLUS : '+';
284
+ MUL : '*';
285
+ ASSIGN : '=';
286
+ WS : ' '+;
287
+ END
288
+
289
+ def rewrite(input)
290
+ lexer = SimpleRewriting2::Lexer.new(input)
291
+ ANTLR3::TokenRewriteStream.new(lexer)
292
+ end
293
+
294
+ example 'rendering over a range' do
295
+ stream = rewrite 'x = 3 * 0;'
296
+ stream.replace 4, 8, '0'
297
+ stream.original_string.should == 'x = 3 * 0;'
298
+ stream.render.should == 'x = 0;'
299
+ stream.render(0, 9).should == 'x = 0;'
300
+ stream.render(4, 8).should == '0'
301
+ end
302
+
303
+ example 'more rendering over a range' do
304
+ stream = rewrite 'x = 3 * 0 + 2 * 0;'
305
+ stream.original_string.should == 'x = 3 * 0 + 2 * 0;'
306
+ stream.replace 4, 8, '0'
307
+ stream.render.should == 'x = 0 + 2 * 0;'
308
+ stream.render(0, 17).should == 'x = 0 + 2 * 0;'
309
+ stream.render(4, 8).should == '0'
310
+ stream.render(0, 8).should == 'x = 0'
311
+ stream.render(12, 16).should == '2 * 0'
312
+ stream.insert_after(17, '// comment')
313
+ stream.render(12, 17).should == '2 * 0;// comment'
314
+ stream.render(0, 8).should == 'x = 0'
315
+ end
316
+
317
+ end
318
+
@@ -0,0 +1,100 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestRewritingWhileParsing < ANTLR3::Test::Functional
7
+
8
+ inline_grammar(<<-'END')
9
+ grammar TokenRewrites;
10
+ options { language = Ruby; }
11
+
12
+ program
13
+ @after {
14
+ @input.insert_before($start,"public class Wrapper {\n")
15
+ @input.insert_after($stop, "\n}\n")
16
+ }
17
+ : method+
18
+ ;
19
+
20
+ method
21
+ : m='method' ID '(' ')' body
22
+ {@input.replace($m, "public void");}
23
+ ;
24
+
25
+ body
26
+ scope {
27
+ decls
28
+ }
29
+ @init {
30
+ $body::decls = []
31
+ }
32
+ : lcurly='{' stat* '}'
33
+ {
34
+ $body::decls.uniq!
35
+ for it in $body::decls
36
+ @input.insert_after($lcurly, "\nint "+it+";")
37
+ end
38
+ }
39
+ ;
40
+
41
+ stat: ID '=' expr ';' {$body::decls << $ID.text.to_s}
42
+ ;
43
+
44
+ expr: mul ('+' mul)*
45
+ ;
46
+
47
+ mul : atom ('*' atom)*
48
+ ;
49
+
50
+ atom: ID
51
+ | INT
52
+ ;
53
+
54
+ ID : ('a'..'z'|'A'..'Z')+ ;
55
+
56
+ INT : ('0'..'9')+ ;
57
+
58
+ WS : (' '|'\t'|'\n')+ {$channel=HIDDEN;}
59
+ ;
60
+ END
61
+
62
+ example 'using a TokenRewriteStream to rewrite input text while parsing' do
63
+ input = <<-END.fixed_indent(0)
64
+ method foo() {
65
+ i = 3;
66
+ k = i;
67
+ i = k*4;
68
+ }
69
+
70
+ method bar() {
71
+ j = i*2;
72
+ }
73
+ END
74
+ expected_output = <<-END.fixed_indent(0).strip!
75
+ public class Wrapper {
76
+ public void foo() {
77
+ int k;
78
+ int i;
79
+ i = 3;
80
+ k = i;
81
+ i = k*4;
82
+ }
83
+
84
+ public void bar() {
85
+ int j;
86
+ j = i*2;
87
+ }
88
+ }
89
+ END
90
+
91
+ lexer = TokenRewrites::Lexer.new( input )
92
+ tokens = ANTLR3::TokenRewriteStream.new( lexer )
93
+ parser = TokenRewrites::Parser.new( tokens )
94
+ parser.program
95
+
96
+ tokens.render.strip.should == expected_output
97
+ end
98
+
99
+ end
100
+
@@ -0,0 +1,750 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestTreeParser1 < ANTLR3::Test::Functional
7
+
8
+ example "flat list" do
9
+ compile_and_load inline_grammar(<<-'END')
10
+ grammar FlatList;
11
+ options {
12
+ language=Ruby;
13
+ output=AST;
14
+ }
15
+ a : ID INT;
16
+ ID : 'a'..'z'+ ;
17
+ INT : '0'..'9'+;
18
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
19
+ END
20
+ compile_and_load inline_grammar(<<-'END')
21
+ tree grammar FlatListWalker;
22
+ options {
23
+ language=Ruby;
24
+ ASTLabelType=CommonTree;
25
+ }
26
+ @members { include ANTLR3::Test::CaptureOutput }
27
+ a : ID INT
28
+ {self.capture("\%s, \%s" \% [$ID, $INT])}
29
+ ;
30
+ END
31
+
32
+ lexer = FlatList::Lexer.new( "abc 34" )
33
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
34
+ parser = FlatList::Parser.new( tokens )
35
+
36
+ result = parser.a
37
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
38
+ nodes.token_stream = tokens
39
+ walker = FlatListWalker::TreeParser.new( nodes )
40
+ walker.a
41
+ walker.output.should == "abc, 34"
42
+ end
43
+
44
+ example "simple tree" do
45
+ compile_and_load inline_grammar(<<-'END')
46
+ grammar SimpleTree;
47
+ options {
48
+ language=Ruby;
49
+ output=AST;
50
+ }
51
+ a : ID INT -> ^(ID INT);
52
+ ID : 'a'..'z'+ ;
53
+ INT : '0'..'9'+;
54
+ WS : (' '|'\\n') {$channel=HIDDEN;} ;
55
+ END
56
+ compile_and_load inline_grammar(<<-'END')
57
+ tree grammar SimpleTreeWalker;
58
+ options {
59
+ language=Ruby;
60
+ ASTLabelType=CommonTree;
61
+ }
62
+ @members { include ANTLR3::Test::CaptureOutput }
63
+
64
+ a : ^(ID INT)
65
+ {capture('\%s, \%s' \% [$ID, $INT])}
66
+ ;
67
+ END
68
+ lexer = SimpleTree::Lexer.new( "abc 34" )
69
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
70
+ parser = SimpleTree::Parser.new( tokens )
71
+
72
+ result = parser.a
73
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
74
+ nodes.token_stream = tokens
75
+ walker = SimpleTreeWalker::TreeParser.new( nodes )
76
+ walker.a
77
+ walker.output.should == "abc, 34"
78
+ end
79
+
80
+ example "flat vs tree decision" do
81
+ compile_and_load inline_grammar(<<-'END')
82
+ grammar FlatVsTreeDecision;
83
+ options {
84
+ language=Ruby;
85
+ output=AST;
86
+ }
87
+ a : b c ;
88
+ b : ID INT -> ^(ID INT);
89
+ c : ID INT;
90
+ ID : 'a'..'z'+ ;
91
+ INT : '0'..'9'+;
92
+ WS : (' '|'\\n') {$channel=HIDDEN;} ;
93
+ END
94
+ compile_and_load inline_grammar(<<-'END')
95
+ tree grammar FlatVsTreeDecisionWalker;
96
+ options {
97
+ language=Ruby;
98
+ ASTLabelType=CommonTree;
99
+ }
100
+ @members { include ANTLR3::Test::CaptureOutput }
101
+
102
+ a : b b ;
103
+ b : ID INT {capture("\%s \%s\n" \% [$ID, $INT])}
104
+ | ^(ID INT) {capture("^(\%s \%s)" \% [$ID, $INT])}
105
+ ;
106
+ END
107
+ lexer = FlatVsTreeDecision::Lexer.new( "a 1 b 2" )
108
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
109
+ parser = FlatVsTreeDecision::Parser.new( tokens )
110
+
111
+ result = parser.a
112
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
113
+ nodes.token_stream = tokens
114
+ walker = FlatVsTreeDecisionWalker::TreeParser.new( nodes )
115
+ walker.a
116
+ walker.output.should == "^(a 1)b 2\n"
117
+ end
118
+
119
+ example "flat vs tree decision2" do
120
+ compile_and_load inline_grammar(<<-'END')
121
+ grammar FlatVsTreeDecision2;
122
+ options {
123
+ language=Ruby;
124
+ output=AST;
125
+ }
126
+ a : b c ;
127
+ b : ID INT+ -> ^(ID INT+);
128
+ c : ID INT+;
129
+ ID : 'a'..'z'+ ;
130
+ INT : '0'..'9'+;
131
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
132
+ END
133
+ compile_and_load inline_grammar(<<-'END')
134
+ tree grammar FlatVsTreeDecision2Walker;
135
+ options {
136
+ language=Ruby;
137
+ ASTLabelType=CommonTree;
138
+ }
139
+ @members { include ANTLR3::Test::CaptureOutput }
140
+ a : b b ;
141
+ b : ID INT+ {say("#{$ID} #{$INT}")}
142
+ | ^(x=ID (y=INT)+) {capture("^(#{$x} #{$y})")}
143
+ ;
144
+ END
145
+ lexer = FlatVsTreeDecision2::Lexer.new( "a 1 2 3 b 4 5" )
146
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
147
+ parser = FlatVsTreeDecision2::Parser.new( tokens )
148
+
149
+ result = parser.a
150
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
151
+ nodes.token_stream = tokens
152
+ walker = FlatVsTreeDecision2Walker::TreeParser.new( nodes )
153
+ walker.a
154
+ walker.output.should == "^(a 3)b 5\n"
155
+ end
156
+
157
+ example "cyclic dfa lookahead" do
158
+ compile_and_load inline_grammar(<<-'END')
159
+ grammar CyclicDFALookahead;
160
+ options {
161
+ language=Ruby;
162
+ output=AST;
163
+ }
164
+ a : ID INT+ PERIOD;
165
+ ID : 'a'..'z'+ ;
166
+ INT : '0'..'9'+;
167
+ SEMI : ';' ;
168
+ PERIOD : '.' ;
169
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
170
+ END
171
+ compile_and_load inline_grammar(<<-'END')
172
+ tree grammar CyclicDFALookaheadWalker;
173
+ options {
174
+ language=Ruby;
175
+ ASTLabelType=CommonTree;
176
+ }
177
+ @members { include ANTLR3::Test::CaptureOutput }
178
+ a : ID INT+ PERIOD {capture("alt 1")}
179
+ | ID INT+ SEMI {capture("alt 2")}
180
+ ;
181
+ END
182
+ lexer = CyclicDFALookahead::Lexer.new( "a 1 2 3." )
183
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
184
+ parser = CyclicDFALookahead::Parser.new( tokens )
185
+
186
+ result = parser.a
187
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
188
+ nodes.token_stream = tokens
189
+ walker = CyclicDFALookaheadWalker::TreeParser.new( nodes )
190
+ walker.a
191
+ walker.output.should == "alt 1"
192
+ end
193
+
194
+ example "nullable child list" do
195
+ compile_and_load inline_grammar(<<-'END')
196
+ grammar NullableChildList;
197
+ options {
198
+ language=Ruby;
199
+ output=AST;
200
+ }
201
+ a : ID INT? -> ^(ID INT?);
202
+ ID : 'a'..'z'+ ;
203
+ INT : '0'..'9'+;
204
+ WS : (' '|'\\n') {$channel=HIDDEN;} ;
205
+ END
206
+ compile_and_load inline_grammar(<<-'END')
207
+ tree grammar NullableChildListWalker;
208
+ options {
209
+ language=Ruby;
210
+ ASTLabelType=CommonTree;
211
+ }
212
+ @members { include ANTLR3::Test::CaptureOutput }
213
+ a : ^(ID INT?)
214
+ {capture($ID.to_s)}
215
+ ;
216
+ END
217
+ lexer = NullableChildList::Lexer.new( "abc" )
218
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
219
+ parser = NullableChildList::Parser.new( tokens )
220
+
221
+ result = parser.a
222
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
223
+ nodes.token_stream = tokens
224
+ walker = NullableChildListWalker::TreeParser.new( nodes )
225
+ walker.a
226
+ walker.output.should == "abc"
227
+ end
228
+
229
+ example "nullable child list2" do
230
+ compile_and_load inline_grammar(<<-'END')
231
+ grammar NullableChildList2;
232
+ options {
233
+ language=Ruby;
234
+ output=AST;
235
+ }
236
+ a : ID INT? SEMI -> ^(ID INT?) SEMI ;
237
+ ID : 'a'..'z'+ ;
238
+ INT : '0'..'9'+;
239
+ SEMI : ';' ;
240
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
241
+ END
242
+ compile_and_load inline_grammar(<<-'END')
243
+ tree grammar NullableChildList2Walker;
244
+ options {
245
+ language=Ruby;
246
+ ASTLabelType=CommonTree;
247
+ }
248
+ @members { include ANTLR3::Test::CaptureOutput }
249
+ a : ^(ID INT?) SEMI
250
+ {capture($ID.to_s)}
251
+ ;
252
+ END
253
+ lexer = NullableChildList2::Lexer.new( "abc;" )
254
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
255
+ parser = NullableChildList2::Parser.new( tokens )
256
+
257
+ result = parser.a
258
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
259
+ nodes.token_stream = tokens
260
+ walker = NullableChildList2Walker::TreeParser.new( nodes )
261
+ walker.a
262
+ walker.output.should == "abc"
263
+ end
264
+
265
+ example "nullable child list3" do
266
+ compile_and_load inline_grammar(<<-'END')
267
+ grammar NullableChildList3;
268
+ options {
269
+ language=Ruby;
270
+ output=AST;
271
+ }
272
+ a : x=ID INT? (y=ID)? SEMI -> ^($x INT? $y?) SEMI ;
273
+ ID : 'a'..'z'+ ;
274
+ INT : '0'..'9'+;
275
+ SEMI : ';' ;
276
+ WS : (' '|'\\n') {$channel=HIDDEN;} ;
277
+ END
278
+ compile_and_load inline_grammar(<<-'END')
279
+ tree grammar NullableChildList3Walker;
280
+ options {
281
+ language=Ruby;
282
+ ASTLabelType=CommonTree;
283
+ }
284
+ @members { include ANTLR3::Test::CaptureOutput }
285
+ a : ^(ID INT? b) SEMI
286
+ {self.capture($ID.to_s + ", " + $b.text.to_s)}
287
+ ;
288
+ b : ID? ;
289
+ END
290
+ lexer = NullableChildList3::Lexer.new( "abc def;" )
291
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
292
+ parser = NullableChildList3::Parser.new( tokens )
293
+
294
+ result = parser.a
295
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
296
+ nodes.token_stream = tokens
297
+ walker = NullableChildList3Walker::TreeParser.new( nodes )
298
+ walker.a
299
+ walker.output.should == "abc, def"
300
+ end
301
+
302
+ example "actions after root" do
303
+ compile_and_load inline_grammar(<<-'END')
304
+ grammar ActionsAfterRoot;
305
+ options {
306
+ language=Ruby;
307
+ output=AST;
308
+ }
309
+ a : x=ID INT? SEMI -> ^($x INT?) ;
310
+ ID : 'a'..'z'+ ;
311
+ INT : '0'..'9'+;
312
+ SEMI : ';' ;
313
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
314
+ END
315
+ compile_and_load inline_grammar(<<-'END')
316
+ tree grammar ActionsAfterRootWalker;
317
+ options {
318
+ language=Ruby;
319
+ ASTLabelType=CommonTree;
320
+ }
321
+ @members { include ANTLR3::Test::CaptureOutput }
322
+ a @init {x=0} : ^(ID {x=1} {x=2} INT?)
323
+ {say( $ID.to_s + ", " + x.to_s )}
324
+ ;
325
+ END
326
+ lexer = ActionsAfterRoot::Lexer.new( "abc;" )
327
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
328
+ parser = ActionsAfterRoot::Parser.new( tokens )
329
+
330
+ result = parser.a
331
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
332
+ nodes.token_stream = tokens
333
+ walker = ActionsAfterRootWalker::TreeParser.new( nodes )
334
+ walker.a
335
+ walker.output.should == "abc, 2\n"
336
+ end
337
+
338
+ example "wildcard lookahead" do
339
+ compile_and_load inline_grammar(<<-'END')
340
+ grammar WildcardLookahead;
341
+ options {language=Ruby; output=AST;}
342
+ a : ID '+'^ INT;
343
+ ID : 'a'..'z'+ ;
344
+ INT : '0'..'9'+;
345
+ SEMI : ';' ;
346
+ PERIOD : '.' ;
347
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
348
+ END
349
+ compile_and_load inline_grammar(<<-'END')
350
+ tree grammar WildcardLookaheadWalker;
351
+ options {language=Ruby; tokenVocab=WildcardLookahead; ASTLabelType=CommonTree;}
352
+ @members { include ANTLR3::Test::CaptureOutput }
353
+ a : ^('+' . INT) { capture("alt 1") }
354
+ ;
355
+ END
356
+ lexer = WildcardLookahead::Lexer.new( "a + 2" )
357
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
358
+ parser = WildcardLookahead::Parser.new( tokens )
359
+
360
+ result = parser.a
361
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
362
+ nodes.token_stream = tokens
363
+ walker = WildcardLookaheadWalker::TreeParser.new( nodes )
364
+ walker.a
365
+ walker.output.should == "alt 1"
366
+ end
367
+
368
+ example "wildcard lookahead2" do
369
+ compile_and_load inline_grammar(<<-'END')
370
+ grammar WildcardLookahead2;
371
+ options {language=Ruby; output=AST;}
372
+ a : ID '+'^ INT;
373
+ ID : 'a'..'z'+ ;
374
+ INT : '0'..'9'+;
375
+ SEMI : ';' ;
376
+ PERIOD : '.' ;
377
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
378
+ END
379
+ compile_and_load inline_grammar(<<-'END')
380
+ tree grammar WildcardLookahead2Walker;
381
+ options {language=Ruby; tokenVocab=WildcardLookahead2; ASTLabelType=CommonTree;}
382
+ @members { include ANTLR3::Test::CaptureOutput }
383
+ a : ^('+' . INT) { capture("alt 1") }
384
+ | ^('+' . .) { capture("alt 2") }
385
+ ;
386
+ END
387
+ lexer = WildcardLookahead2::Lexer.new( "a + 2" )
388
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
389
+ parser = WildcardLookahead2::Parser.new( tokens )
390
+
391
+ result = parser.a
392
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
393
+ nodes.token_stream = tokens
394
+ walker = WildcardLookahead2Walker::TreeParser.new( nodes )
395
+ walker.a
396
+ walker.output.should == "alt 1"
397
+ end
398
+
399
+ example "wildcard lookahead3" do
400
+ compile_and_load inline_grammar(<<-'END')
401
+ grammar WildcardLookahead3;
402
+ options {language=Ruby; output=AST;}
403
+ a : ID '+'^ INT;
404
+ ID : 'a'..'z'+ ;
405
+ INT : '0'..'9'+;
406
+ SEMI : ';' ;
407
+ PERIOD : '.' ;
408
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
409
+ END
410
+ compile_and_load inline_grammar(<<-'END')
411
+ tree grammar WildcardLookahead3Walker;
412
+ options {language=Ruby; tokenVocab=WildcardLookahead3; ASTLabelType=CommonTree;}
413
+ @members { include ANTLR3::Test::CaptureOutput }
414
+ a : ^('+' ID INT) { capture("alt 1") }
415
+ | ^('+' . .) { capture("alt 2") }
416
+ ;
417
+ END
418
+ lexer = WildcardLookahead3::Lexer.new( "a + 2" )
419
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
420
+ parser = WildcardLookahead3::Parser.new( tokens )
421
+
422
+ result = parser.a
423
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
424
+ nodes.token_stream = tokens
425
+ walker = WildcardLookahead3Walker::TreeParser.new( nodes )
426
+ walker.a
427
+ walker.output.should == "alt 1"
428
+ end
429
+
430
+ example "wildcard plus lookahead" do
431
+ compile_and_load inline_grammar(<<-'END')
432
+ grammar WildcardPlusLookahead;
433
+ options {language=Ruby; output=AST;}
434
+ a : ID '+'^ INT;
435
+ ID : 'a'..'z'+ ;
436
+ INT : '0'..'9'+;
437
+ SEMI : ';' ;
438
+ PERIOD : '.' ;
439
+ WS : (' '|'\n') {$channel=HIDDEN;} ;
440
+ END
441
+ compile_and_load inline_grammar(<<-'END')
442
+ tree grammar WildcardPlusLookaheadWalker;
443
+ options {language=Ruby; tokenVocab=WildcardPlusLookahead; ASTLabelType=CommonTree;}
444
+ @members { include ANTLR3::Test::CaptureOutput }
445
+ a : ^('+' INT INT ) { capture("alt 1") }
446
+ | ^('+' .+) { capture("alt 2") }
447
+ ;
448
+ END
449
+ lexer = WildcardPlusLookahead::Lexer.new( "a + 2" )
450
+ tokens = ANTLR3::CommonTokenStream.new( lexer )
451
+ parser = WildcardPlusLookahead::Parser.new( tokens )
452
+
453
+ result = parser.a
454
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
455
+ nodes.token_stream = tokens
456
+ walker = WildcardPlusLookaheadWalker::TreeParser.new( nodes )
457
+ walker.a
458
+ walker.output.should == "alt 2"
459
+ end
460
+
461
+ end
462
+
463
+
464
+ class TestTreeParser2 < ANTLR3::Test::Functional
465
+ inline_grammar(<<-'END')
466
+ grammar GenericLanguage;
467
+ options {
468
+ language = Ruby;
469
+ output=AST;
470
+ }
471
+
472
+ tokens {
473
+ VAR_DEF;
474
+ ARG_DEF;
475
+ FUNC_HDR;
476
+ FUNC_DECL;
477
+ FUNC_DEF;
478
+ BLOCK;
479
+ }
480
+
481
+ program
482
+ : declaration+
483
+ ;
484
+
485
+ declaration
486
+ : variable
487
+ | functionHeader ';' -> ^(FUNC_DECL functionHeader)
488
+ | functionHeader block -> ^(FUNC_DEF functionHeader block)
489
+ ;
490
+
491
+ variable
492
+ : type declarator ';' -> ^(VAR_DEF type declarator)
493
+ ;
494
+
495
+ declarator
496
+ : ID
497
+ ;
498
+
499
+ functionHeader
500
+ : type ID '(' ( formalParameter ( ',' formalParameter )* )? ')'
501
+ -> ^(FUNC_HDR type ID formalParameter+)
502
+ ;
503
+
504
+ formalParameter
505
+ : type declarator -> ^(ARG_DEF type declarator)
506
+ ;
507
+
508
+ type
509
+ : 'int'
510
+ | 'char'
511
+ | 'void'
512
+ | ID
513
+ ;
514
+
515
+ block
516
+ : lc='{'
517
+ variable*
518
+ stat*
519
+ '}'
520
+ -> ^(BLOCK[$lc,"BLOCK"] variable* stat*)
521
+ ;
522
+
523
+ stat: forStat
524
+ | expr ';'!
525
+ | block
526
+ | assignStat ';'!
527
+ | ';'!
528
+ ;
529
+
530
+ forStat
531
+ : 'for' '(' start=assignStat ';' expr ';' next=assignStat ')' block
532
+ -> ^('for' $start expr $next block)
533
+ ;
534
+
535
+ assignStat
536
+ : ID EQ expr -> ^(EQ ID expr)
537
+ ;
538
+
539
+ expr: condExpr
540
+ ;
541
+
542
+ condExpr
543
+ : aexpr ( ('=='^ | '<'^) aexpr )?
544
+ ;
545
+
546
+ aexpr
547
+ : atom ( '+'^ atom )*
548
+ ;
549
+
550
+ atom
551
+ : ID
552
+ | INT
553
+ | '(' expr ')' -> expr
554
+ ;
555
+
556
+ FOR : 'for' ;
557
+ INT_TYPE : 'int' ;
558
+ CHAR: 'char';
559
+ VOID: 'void';
560
+
561
+ ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
562
+ ;
563
+
564
+ INT : ('0'..'9')+
565
+ ;
566
+
567
+ EQ : '=' ;
568
+ EQEQ : '==' ;
569
+ LT : '<' ;
570
+ PLUS : '+' ;
571
+
572
+ WS : ( ' '
573
+ | '\t'
574
+ | '\r'
575
+ | '\n'
576
+ )+
577
+ { $channel=HIDDEN }
578
+ ;
579
+ END
580
+
581
+ inline_grammar(<<-'END')
582
+ tree grammar GenericLanguageWalker;
583
+ options {
584
+ language = Ruby;
585
+ tokenVocab = GenericLanguage;
586
+ ASTLabelType = CommonTree;
587
+ }
588
+
589
+ @init { @traces = [] }
590
+ @members {
591
+ attr_reader :traces
592
+
593
+ def trace_in(rule_name, rule_index)
594
+ @traces << ">#{rule_name}"
595
+ end
596
+
597
+ def trace_out(rule_name, rule_index)
598
+ @traces << "<#{rule_name}"
599
+ end
600
+ }
601
+
602
+ program
603
+ : declaration+
604
+ ;
605
+
606
+ declaration
607
+ : variable
608
+ | ^(FUNC_DECL functionHeader)
609
+ | ^(FUNC_DEF functionHeader block)
610
+ ;
611
+
612
+ variable returns [res]
613
+ : ^(VAR_DEF type declarator)
614
+ {
615
+ $res = $declarator.text;
616
+ }
617
+ ;
618
+
619
+ declarator
620
+ : ID
621
+ ;
622
+
623
+ functionHeader
624
+ : ^(FUNC_HDR type ID formalParameter+)
625
+ ;
626
+
627
+ formalParameter
628
+ : ^(ARG_DEF type declarator)
629
+ ;
630
+
631
+ type
632
+ : 'int'
633
+ | 'char'
634
+ | 'void'
635
+ | ID
636
+ ;
637
+
638
+ block
639
+ : ^(BLOCK variable* stat*)
640
+ ;
641
+
642
+ stat: forStat
643
+ | expr
644
+ | block
645
+ ;
646
+
647
+ forStat
648
+ : ^('for' expr expr expr block)
649
+ ;
650
+
651
+ expr: ^(EQEQ expr expr)
652
+ | ^(LT expr expr)
653
+ | ^(PLUS expr expr)
654
+ | ^(EQ ID expr)
655
+ | atom
656
+ ;
657
+
658
+ atom
659
+ : ID
660
+ | INT
661
+ ;
662
+ END
663
+
664
+ compile_options :trace => true
665
+
666
+ example "processing AST output from a parser with a tree parser" do
667
+ input_source = <<-END.fixed_indent(0)
668
+ char c;
669
+ int x;
670
+
671
+ void bar(int x);
672
+
673
+ int foo(int y, char d) {
674
+ int i;
675
+ for (i=0; i<3; i=i+1) {
676
+ x=3;
677
+ y=5;
678
+ }
679
+ }
680
+ END
681
+
682
+ lexer = GenericLanguage::Lexer.new( input_source )
683
+ parser = GenericLanguage::Parser.new( lexer )
684
+
685
+ expected_tree = <<-END.strip!.gsub!(/\s+/, ' ')
686
+ (VAR_DEF char c)
687
+ (VAR_DEF int x)
688
+ (FUNC_DECL (FUNC_HDR void bar (ARG_DEF int x)))
689
+ (FUNC_DEF
690
+ (FUNC_HDR int foo (ARG_DEF int y) (ARG_DEF char d))
691
+ (BLOCK
692
+ (VAR_DEF int i)
693
+ (for (= i 0) (< i 3) (= i (+ i 1))
694
+ (BLOCK (= x 3) (= y 5)))))
695
+ END
696
+
697
+ result = parser.program
698
+ result.tree.inspect.should == expected_tree
699
+
700
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
701
+ nodes.token_stream = parser.input
702
+ tree_parser = GenericLanguageWalker::TreeParser.new( nodes )
703
+
704
+ tree_parser.program
705
+ tree_parser.traces.should == %w(
706
+ >program >declaration >variable >type
707
+ <type >declarator <declarator <variable
708
+ <declaration >declaration >variable >type
709
+ <type >declarator <declarator <variable
710
+ <declaration >declaration >functionHeader >type
711
+ <type >formalParameter >type <type
712
+ >declarator <declarator <formalParameter <functionHeader
713
+ <declaration >declaration >functionHeader >type
714
+ <type >formalParameter >type <type
715
+ >declarator <declarator <formalParameter >formalParameter
716
+ >type <type >declarator <declarator
717
+ <formalParameter <functionHeader >block >variable
718
+ >type <type >declarator <declarator
719
+ <variable >stat >forStat >expr
720
+ >expr >atom <atom <expr
721
+ <expr >expr >expr >atom
722
+ <atom <expr >expr >atom
723
+ <atom <expr <expr >expr
724
+ >expr >expr >atom <atom
725
+ <expr >expr >atom <atom
726
+ <expr <expr <expr >block
727
+ >stat >expr >expr >atom
728
+ <atom <expr <expr <stat
729
+ >stat >expr >expr >atom
730
+ <atom <expr <expr <stat
731
+ <block <forStat <stat <block
732
+ <declaration <program
733
+ )
734
+ end
735
+
736
+ example 'tree parser rule label property references' do
737
+ input = "char c;\n"
738
+ lexer = GenericLanguage::Lexer.new( "char c;\n" )
739
+ parser = GenericLanguage::Parser.new( lexer )
740
+
741
+ result = parser.variable
742
+ nodes = ANTLR3::AST::CommonTreeNodeStream.new( result.tree )
743
+ nodes.token_stream = parser.input
744
+
745
+ tree_parser = GenericLanguageWalker::TreeParser.new( nodes )
746
+ tree_parser.variable.should == 'c'
747
+ end
748
+
749
+ end
750
+