antlr3 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/ANTLR-LICENSE.txt +26 -0
  2. data/History.txt +66 -0
  3. data/README.txt +139 -0
  4. data/bin/antlr4ruby +33 -0
  5. data/java/RubyTarget.java +524 -0
  6. data/java/antlr-full-3.2.1.jar +0 -0
  7. data/lib/antlr3.rb +176 -0
  8. data/lib/antlr3/constants.rb +88 -0
  9. data/lib/antlr3/debug.rb +701 -0
  10. data/lib/antlr3/debug/event-hub.rb +210 -0
  11. data/lib/antlr3/debug/record-event-listener.rb +25 -0
  12. data/lib/antlr3/debug/rule-tracer.rb +55 -0
  13. data/lib/antlr3/debug/socket.rb +360 -0
  14. data/lib/antlr3/debug/trace-event-listener.rb +92 -0
  15. data/lib/antlr3/dfa.rb +247 -0
  16. data/lib/antlr3/dot.rb +174 -0
  17. data/lib/antlr3/error.rb +657 -0
  18. data/lib/antlr3/main.rb +561 -0
  19. data/lib/antlr3/modes/ast-builder.rb +41 -0
  20. data/lib/antlr3/modes/filter.rb +56 -0
  21. data/lib/antlr3/profile.rb +322 -0
  22. data/lib/antlr3/recognizers.rb +1280 -0
  23. data/lib/antlr3/streams.rb +985 -0
  24. data/lib/antlr3/streams/interactive.rb +91 -0
  25. data/lib/antlr3/streams/rewrite.rb +412 -0
  26. data/lib/antlr3/test/call-stack.rb +57 -0
  27. data/lib/antlr3/test/config.rb +23 -0
  28. data/lib/antlr3/test/core-extensions.rb +269 -0
  29. data/lib/antlr3/test/diff.rb +165 -0
  30. data/lib/antlr3/test/functional.rb +207 -0
  31. data/lib/antlr3/test/grammar.rb +371 -0
  32. data/lib/antlr3/token.rb +592 -0
  33. data/lib/antlr3/tree.rb +1415 -0
  34. data/lib/antlr3/tree/debug.rb +163 -0
  35. data/lib/antlr3/tree/visitor.rb +84 -0
  36. data/lib/antlr3/tree/wizard.rb +481 -0
  37. data/lib/antlr3/util.rb +149 -0
  38. data/lib/antlr3/version.rb +27 -0
  39. data/samples/ANTLRv3Grammar.g +621 -0
  40. data/samples/Cpp.g +749 -0
  41. data/templates/AST.stg +335 -0
  42. data/templates/ASTDbg.stg +40 -0
  43. data/templates/ASTParser.stg +153 -0
  44. data/templates/ASTTreeParser.stg +272 -0
  45. data/templates/Dbg.stg +192 -0
  46. data/templates/Ruby.stg +1514 -0
  47. data/test/functional/ast-output/auto-ast.rb +797 -0
  48. data/test/functional/ast-output/construction.rb +555 -0
  49. data/test/functional/ast-output/hetero-nodes.rb +753 -0
  50. data/test/functional/ast-output/rewrites.rb +1327 -0
  51. data/test/functional/ast-output/tree-rewrite.rb +1662 -0
  52. data/test/functional/debugging/debug-mode.rb +689 -0
  53. data/test/functional/debugging/profile-mode.rb +165 -0
  54. data/test/functional/debugging/rule-tracing.rb +74 -0
  55. data/test/functional/delegation/import.rb +379 -0
  56. data/test/functional/lexer/basic.rb +559 -0
  57. data/test/functional/lexer/filter-mode.rb +245 -0
  58. data/test/functional/lexer/nuances.rb +47 -0
  59. data/test/functional/lexer/properties.rb +104 -0
  60. data/test/functional/lexer/syn-pred.rb +32 -0
  61. data/test/functional/lexer/xml.rb +206 -0
  62. data/test/functional/main/main-scripts.rb +245 -0
  63. data/test/functional/parser/actions.rb +224 -0
  64. data/test/functional/parser/backtracking.rb +244 -0
  65. data/test/functional/parser/basic.rb +282 -0
  66. data/test/functional/parser/calc.rb +98 -0
  67. data/test/functional/parser/ll-star.rb +143 -0
  68. data/test/functional/parser/nuances.rb +165 -0
  69. data/test/functional/parser/predicates.rb +103 -0
  70. data/test/functional/parser/properties.rb +242 -0
  71. data/test/functional/parser/rule-methods.rb +132 -0
  72. data/test/functional/parser/scopes.rb +274 -0
  73. data/test/functional/token-rewrite/basic.rb +318 -0
  74. data/test/functional/token-rewrite/via-parser.rb +100 -0
  75. data/test/functional/tree-parser/basic.rb +750 -0
  76. data/test/unit/sample-input/file-stream-1 +2 -0
  77. data/test/unit/sample-input/teststreams.input2 +2 -0
  78. data/test/unit/test-dfa.rb +52 -0
  79. data/test/unit/test-exceptions.rb +44 -0
  80. data/test/unit/test-recognizers.rb +55 -0
  81. data/test/unit/test-scheme.rb +62 -0
  82. data/test/unit/test-streams.rb +459 -0
  83. data/test/unit/test-tree-wizard.rb +535 -0
  84. data/test/unit/test-trees.rb +854 -0
  85. metadata +205 -0
@@ -0,0 +1,242 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestRulePropertyReference < ANTLR3::Test::Functional
7
+
8
+ inline_grammar(<<-'END')
9
+ grammar RuleProperties;
10
+ options { language = Ruby; }
11
+
12
+ @parser::members {
13
+ def emit_error_message(msg)
14
+ # do nothing
15
+ end
16
+ def report_error(error)
17
+ raise error
18
+ end
19
+ }
20
+
21
+ @lexer::members {
22
+ def emit_error_message(msg)
23
+ # do nothing
24
+ end
25
+ def report_error(error)
26
+ raise error
27
+ end
28
+ }
29
+
30
+ a returns [bla]
31
+ @after { $bla = [$start, $stop, $text] }
32
+ : A+
33
+ ;
34
+
35
+ A: 'a'..'z';
36
+
37
+ WS: ' '+ { $channel = HIDDEN };
38
+ END
39
+
40
+ example "accessing rule properties" do
41
+ lexer = RuleProperties::Lexer.new( ' a a a a ' )
42
+ parser = RuleProperties::Parser.new lexer
43
+ start, stop, text = parser.a.bla
44
+
45
+ start.index.should == 1
46
+ stop.index.should == 7
47
+ text.should == 'a a a a'
48
+ end
49
+
50
+
51
+ end
52
+
53
+ class TestLabels < ANTLR3::Test::Functional
54
+
55
+ inline_grammar(<<-'END')
56
+ grammar Labels;
57
+ options { language = Ruby; }
58
+
59
+ @parser::members {
60
+ def recover(e)
61
+ raise e
62
+ end
63
+ }
64
+
65
+ @lexer::members {
66
+ def recover(e)
67
+ raise e
68
+ end
69
+ }
70
+
71
+ a returns [l]
72
+ : ids+=A ( ',' ids+=(A|B) )* C D w=. ids+=. F EOF
73
+ { $l = [$ids, $w] }
74
+ ;
75
+
76
+ A: 'a'..'z';
77
+ B: '0'..'9';
78
+ C: a='A' { $a };
79
+ D: a='FOOBAR' { $a };
80
+ E: 'GNU' a=. { $a };
81
+ F: 'BLARZ' a=EOF { $a };
82
+
83
+ WS: ' '+ { $channel = HIDDEN };
84
+ END
85
+
86
+ example "parsing 'a, b, c, 1, 2 A FOOBAR GNU1 A BLARZ'" do
87
+ lexer = Labels::Lexer.new 'a, b, c, 1, 2 A FOOBAR GNU1 A BLARZ'
88
+ parser = Labels::Parser.new lexer
89
+ ids, w = parser.a
90
+
91
+ ids.should have(6).things
92
+ ids[0].text.should == 'a'
93
+ ids[1].text.should == 'b'
94
+ ids[2].text.should == 'c'
95
+ ids[3].text.should == '1'
96
+ ids[4].text.should == '2'
97
+ ids[5].text.should == 'A'
98
+
99
+ w.text.should == 'GNU1'
100
+ end
101
+
102
+
103
+ end
104
+
105
+
106
+ class TestTokenLabelReference < ANTLR3::Test::Functional
107
+
108
+ inline_grammar(<<-'END')
109
+ grammar TokenLabels;
110
+ options {
111
+ language = Ruby;
112
+ }
113
+
114
+ @parser::members {
115
+ def emit_error_message(msg)
116
+ # do nothing
117
+ end
118
+ def report_error(error)
119
+ raise error
120
+ end
121
+ }
122
+
123
+ @lexer::members {
124
+ def emit_error_message(msg)
125
+ # do nothing
126
+ end
127
+ def report_error(error)
128
+ raise error
129
+ end
130
+ }
131
+
132
+ a returns [$tk]
133
+ : t=A
134
+ {
135
+ $tk = [
136
+ $t.text,
137
+ $t.type,
138
+ $t.name,
139
+ $t.line,
140
+ $t.pos,
141
+ $t.index,
142
+ $t.channel
143
+ ]
144
+ }
145
+ ;
146
+
147
+ A: 'a'..'z';
148
+
149
+ WS :
150
+ ( ' '
151
+ | '\t'
152
+ | ( '\n'
153
+ | '\r\n'
154
+ | '\r'
155
+ )
156
+ )+
157
+ { $channel = HIDDEN }
158
+ ;
159
+ END
160
+
161
+ example "accessing tokens with labels" do
162
+ lexer = TokenLabels::Lexer.new( ' a' )
163
+ parser = TokenLabels::Parser.new lexer
164
+ tk = parser.a
165
+ tk.should == [
166
+ 'a', TokenLabels::TokenData::A, 'A',
167
+ 1, 3, 1, :default
168
+ ]
169
+ end
170
+
171
+
172
+ end
173
+
174
+ class TestRuleLabelReference < ANTLR3::Test::Functional
175
+
176
+ inline_grammar(<<-'END')
177
+ grammar RuleLabelReference;
178
+ options {language = Ruby;}
179
+
180
+ @parser::members {
181
+ def emit_error_message(msg)
182
+ # do nothing
183
+ end
184
+ def report_error(error)
185
+ raise error
186
+ end
187
+ }
188
+
189
+ @lexer::members {
190
+ def emit_error_message(msg)
191
+ # do nothing
192
+ end
193
+ def report_error(error)
194
+ raise error
195
+ end
196
+ }
197
+
198
+ a returns [bla]: t=b
199
+ {
200
+ $bla = [$t.start, $t.stop, $t.text]
201
+ }
202
+ ;
203
+
204
+ b: A+;
205
+
206
+ A: 'a'..'z';
207
+
208
+ WS: ' '+ { $channel = HIDDEN };
209
+ END
210
+
211
+ example "referencing rule properties using rule labels" do
212
+ lexer = RuleLabelReference::Lexer.new( ' a a a a ' )
213
+ parser = RuleLabelReference::Parser.new lexer
214
+ start, stop, text = parser.a
215
+
216
+ start.index.should == 1
217
+ stop.index.should == 7
218
+ text.should == 'a a a a'
219
+ end
220
+
221
+ end
222
+
223
+
224
+
225
+ class TestReferenceDoesntSetChannel < ANTLR3::Test::Functional
226
+
227
+ inline_grammar(<<-'END')
228
+ grammar ReferenceSetChannel;
229
+ options {language=Ruby;}
230
+ a returns [foo]: A EOF { $foo = '\%s, channel=\%p' \% [$A.text, $A.channel]; } ;
231
+ A : '-' WS I ;
232
+ I : '0'..'9'+ ;
233
+ WS: ' ' | '\t';
234
+ END
235
+
236
+ example 'verifying that a token reference does not set its channel' do
237
+ lexer = ReferenceSetChannel::Lexer.new("- 34")
238
+ parser = ReferenceSetChannel::Parser.new lexer
239
+ parser.a.should == "- 34, channel=:default"
240
+ end
241
+
242
+ end
@@ -0,0 +1,132 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestParameters < ANTLR3::Test::Functional
7
+
8
+ inline_grammar(<<-'END')
9
+ grammar Parameters;
10
+ options {
11
+ language = Ruby;
12
+ }
13
+
14
+ @parser::members {
15
+ def emit_error_message(msg)
16
+ # do nothing
17
+ end
18
+ def report_error(error)
19
+ raise error
20
+ end
21
+ }
22
+
23
+ @lexer::members {
24
+ def emit_error_message(msg)
25
+ # do nothing
26
+ end
27
+ def report_error(error)
28
+ raise error
29
+ end
30
+ }
31
+
32
+ a[arg1, arg2] returns [l]
33
+ : A+ EOF
34
+ {
35
+ l = [$arg1, $arg2]
36
+ $arg1 = "gnarz"
37
+ }
38
+ ;
39
+
40
+ A: 'a'..'z';
41
+
42
+ WS: ' '+ { $channel = HIDDEN };
43
+ END
44
+
45
+ example "rules with method parameters" do
46
+ lexer = Parameters::Lexer.new( 'a a a' )
47
+ parser = Parameters::Parser.new lexer
48
+ r = parser.a( 'foo', 'bar' )
49
+ r.should == %w(foo bar)
50
+ end
51
+
52
+ end
53
+
54
+
55
+ class TestMultipleReturnValues < ANTLR3::Test::Functional
56
+
57
+ inline_grammar(<<-'END')
58
+ grammar MultipleReturnValues;
59
+ options { language = Ruby; }
60
+ @parser::members {
61
+ def emit_error_message(msg)
62
+ # do nothing
63
+ end
64
+ def report_error(error)
65
+ raise error
66
+ end
67
+ }
68
+
69
+ @lexer::members {
70
+ def emit_error_message(msg)
71
+ # do nothing
72
+ end
73
+ def report_error(error)
74
+ raise error
75
+ end
76
+ }
77
+
78
+ a returns [foo, bar]: A
79
+ {
80
+ $foo = "foo";
81
+ $bar = "bar";
82
+ }
83
+ ;
84
+
85
+ A: 'a'..'z';
86
+
87
+ WS :
88
+ ( ' '
89
+ | '\t'
90
+ | ( '\n'
91
+ | '\r\n'
92
+ | '\r'
93
+ )
94
+ )+
95
+ { $channel = HIDDEN }
96
+ ;
97
+ END
98
+
99
+ example "multi-valued rule return structures" do
100
+ lexer = MultipleReturnValues::Lexer.new( ' a' )
101
+ parser = MultipleReturnValues::Parser.new lexer
102
+ ret = parser.a
103
+
104
+ ret.foo.should == 'foo'
105
+ ret.bar.should == 'bar'
106
+ end
107
+
108
+ end
109
+
110
+
111
+ class TestRuleVisibility < ANTLR3::Test::Functional
112
+ inline_grammar(<<-'END')
113
+ grammar RuleVisibility;
114
+ options { language=Ruby; }
115
+
116
+ public a: ID;
117
+ private b: DIGIT;
118
+ protected c: ID DIGIT;
119
+
120
+ DIGIT: ('0'..'9')+;
121
+ ID: ('a'..'z' | 'A'..'Z')+;
122
+ WS: (' ' | '\t' | '\n' | '\r' | '\f')+ { $channel=HIDDEN; };
123
+ END
124
+
125
+ example 'using visibility modifiers on rules' do
126
+ RuleVisibility::Parser.public_instance_methods.should include('a')
127
+ RuleVisibility::Parser.protected_instance_methods.should include('c')
128
+ RuleVisibility::Parser.private_instance_methods.should include('b')
129
+ end
130
+
131
+ end
132
+
@@ -0,0 +1,274 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestScopes1 < ANTLR3::Test::Functional
7
+
8
+ inline_grammar(<<-'END')
9
+ grammar SimpleScope;
10
+
11
+ options {
12
+ language = Ruby;
13
+ }
14
+
15
+ prog
16
+ scope {
17
+ name
18
+ }
19
+ : ID {$prog::name=$ID.text;}
20
+ ;
21
+
22
+ ID : ('a'..'z')+
23
+ ;
24
+
25
+ WS : (' '|'\n'|'\r')+ {$channel=HIDDEN}
26
+ ;
27
+ END
28
+
29
+ example "parsing 'foobar'" do
30
+ lexer = SimpleScope::Lexer.new( 'foobar' )
31
+ parser = SimpleScope::Parser.new lexer
32
+ parser.prog
33
+ end
34
+ end
35
+
36
+ class TestScopes2 < ANTLR3::Test::Functional
37
+ inline_grammar(<<-'END')
38
+ grammar LotsaScopes;
39
+
40
+ options {
41
+ language = Ruby;
42
+ }
43
+
44
+ /* global scopes */
45
+
46
+ scope aScope {
47
+ names;
48
+ }
49
+
50
+ @members {
51
+ def emit_error_message(msg)
52
+ # do nothing
53
+ end
54
+
55
+ def report_error(error)
56
+ raise error
57
+ end
58
+ }
59
+
60
+ a
61
+ scope aScope;
62
+ : {$aScope::names = []} ID*
63
+ ;
64
+
65
+
66
+ /* rule scopes, from the book, final beta, p.147 */
67
+
68
+ b[v]
69
+ scope {x}
70
+ : {$b::x = v;} b2
71
+ ;
72
+
73
+ b2
74
+ : b3
75
+ ;
76
+
77
+ b3
78
+ : {$b::x}?=> ID // only visible, if b was called with True
79
+ | NUM
80
+ ;
81
+
82
+
83
+ /* rule scopes, from the book, final beta, p.148 */
84
+
85
+ c returns [res]
86
+ scope {
87
+ symbols
88
+ }
89
+ @init {
90
+ $c::symbols = Set.new;
91
+ }
92
+ : '{' c1* c2+ '}'
93
+ { $res = $c::symbols; }
94
+ ;
95
+
96
+ c1
97
+ : 'int' ID {$c::symbols.add($ID.text)} ';'
98
+ ;
99
+
100
+ c2
101
+ : ID '=' NUM ';'
102
+ {
103
+ $c::symbols.include?($ID.text) or raise RuntimeError, $ID.text
104
+ }
105
+ ;
106
+
107
+ /* recursive rule scopes, from the book, final beta, p.150 */
108
+
109
+ d returns [res]
110
+ scope {
111
+ symbols
112
+ }
113
+ @init {
114
+ $d::symbols = Set.new
115
+ }
116
+ : '{' d1* d2* '}'
117
+ { $res = $d::symbols; }
118
+ ;
119
+
120
+ d1
121
+ : 'int' ID {$d::symbols.add($ID.text)} ';'
122
+ ;
123
+
124
+ d2
125
+ : ID '=' NUM ';'
126
+ {
127
+ catch(:found) do
128
+ level = ($d.length - 1).downto(0) do |s|
129
+ $d[s].symbols.include?($ID.text) and throw(:found)
130
+ end
131
+ raise $ID.text
132
+ end
133
+ }
134
+ | d
135
+ ;
136
+
137
+ /* recursive rule scopes, access bottom-most scope */
138
+
139
+ e returns [res]
140
+ scope {
141
+ a
142
+ }
143
+ @after {
144
+ $res = $e::a;
145
+ }
146
+ : NUM { $e[0]::a = Integer($NUM.text); }
147
+ | '{' e '}'
148
+ ;
149
+
150
+
151
+ /* recursive rule scopes, access with negative index */
152
+
153
+ f returns [res]
154
+ scope {
155
+ a
156
+ }
157
+ @after {
158
+ $res = $f::a;
159
+ }
160
+ : NUM { $f[-2]::a = Integer($NUM.text); }
161
+ | '{' f '}'
162
+ ;
163
+
164
+
165
+ /* tokens */
166
+
167
+ ID : ('a'..'z')+
168
+ ;
169
+
170
+ NUM : ('0'..'9')+
171
+ ;
172
+
173
+ WS : (' '|'\n'|'\r')+ {$channel=HIDDEN}
174
+ ;
175
+ END
176
+
177
+ example "parsing 'foobar' with rule a" do
178
+ lexer = LotsaScopes::Lexer.new("foobar")
179
+ parser = LotsaScopes::Parser.new lexer
180
+ parser.a
181
+ end
182
+
183
+ example "failing to parse 'foobar' with rule b[false]" do
184
+ lexer = LotsaScopes::Lexer.new("foobar")
185
+ parser = LotsaScopes::Parser.new lexer
186
+ proc { parser.b(false) }.should raise_error(ANTLR3::RecognitionError)
187
+ end
188
+
189
+ example "parsing 'foobar' with rule b[true]" do
190
+ lexer = LotsaScopes::Lexer.new("foobar")
191
+ parser = LotsaScopes::Parser.new lexer
192
+ parser.b(true)
193
+ end
194
+
195
+ example "parsing a decl block with rule c" do
196
+ lexer = LotsaScopes::Lexer.new(<<-END.here_indent!)
197
+ | {
198
+ | int i;
199
+ | int j;
200
+ | i = 0;
201
+ | }
202
+ END
203
+ parser = LotsaScopes::Parser.new lexer
204
+
205
+ symbols = parser.c
206
+ symbols.should have(2).things
207
+ symbols.should include 'i'
208
+ symbols.should include 'j'
209
+ end
210
+
211
+ example "failing to parse undeclared symbols with rule c" do
212
+ lexer = LotsaScopes::Lexer.new(<<-END.here_indent!)
213
+ | {
214
+ | int i;
215
+ | int j;
216
+ | i = 0;
217
+ | x = 4;
218
+ | }
219
+ END
220
+ parser = LotsaScopes::Parser.new lexer
221
+
222
+ proc { parser.c }.should raise_error RuntimeError, 'x'
223
+ end
224
+
225
+ example "parsing nested declaration blocks" do
226
+ lexer = LotsaScopes::Lexer.new(<<-END.here_indent!)
227
+ | {
228
+ | int i;
229
+ | int j;
230
+ | i = 0;
231
+ | {
232
+ | int i;
233
+ | int x;
234
+ | x = 5;
235
+ | }
236
+ | }
237
+ END
238
+ parser = LotsaScopes::Parser.new lexer
239
+
240
+ symbols = parser.d
241
+ symbols.should have(2).things
242
+ symbols.should include 'i'
243
+ symbols.should include 'j'
244
+ end
245
+
246
+ example "parsing a deeply nested set of blocks with rule e" do
247
+ lexer = LotsaScopes::Lexer.new(<<-END.here_indent!)
248
+ | { { { { 12 } } } }
249
+ END
250
+
251
+ parser = LotsaScopes::Parser.new lexer
252
+ parser.e.should == 12
253
+ end
254
+
255
+ example "parsing a deeply nested set of blocks with rule f" do
256
+ lexer = LotsaScopes::Lexer.new(<<-END.here_indent!)
257
+ | { { { { 12 } } } }
258
+ END
259
+
260
+ parser = LotsaScopes::Parser.new lexer
261
+ parser.f.should == nil
262
+ end
263
+
264
+ example "parsing a 2-level nested set of blocks with rule f" do
265
+ lexer = LotsaScopes::Lexer.new(<<-END.here_indent!)
266
+ | { { 12 } }
267
+ END
268
+ parser = LotsaScopes::Parser.new lexer
269
+
270
+ parser.f.should == nil
271
+ end
272
+
273
+ end
274
+