antlr3 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/ANTLR-LICENSE.txt +26 -0
  2. data/History.txt +66 -0
  3. data/README.txt +139 -0
  4. data/bin/antlr4ruby +33 -0
  5. data/java/RubyTarget.java +524 -0
  6. data/java/antlr-full-3.2.1.jar +0 -0
  7. data/lib/antlr3.rb +176 -0
  8. data/lib/antlr3/constants.rb +88 -0
  9. data/lib/antlr3/debug.rb +701 -0
  10. data/lib/antlr3/debug/event-hub.rb +210 -0
  11. data/lib/antlr3/debug/record-event-listener.rb +25 -0
  12. data/lib/antlr3/debug/rule-tracer.rb +55 -0
  13. data/lib/antlr3/debug/socket.rb +360 -0
  14. data/lib/antlr3/debug/trace-event-listener.rb +92 -0
  15. data/lib/antlr3/dfa.rb +247 -0
  16. data/lib/antlr3/dot.rb +174 -0
  17. data/lib/antlr3/error.rb +657 -0
  18. data/lib/antlr3/main.rb +561 -0
  19. data/lib/antlr3/modes/ast-builder.rb +41 -0
  20. data/lib/antlr3/modes/filter.rb +56 -0
  21. data/lib/antlr3/profile.rb +322 -0
  22. data/lib/antlr3/recognizers.rb +1280 -0
  23. data/lib/antlr3/streams.rb +985 -0
  24. data/lib/antlr3/streams/interactive.rb +91 -0
  25. data/lib/antlr3/streams/rewrite.rb +412 -0
  26. data/lib/antlr3/test/call-stack.rb +57 -0
  27. data/lib/antlr3/test/config.rb +23 -0
  28. data/lib/antlr3/test/core-extensions.rb +269 -0
  29. data/lib/antlr3/test/diff.rb +165 -0
  30. data/lib/antlr3/test/functional.rb +207 -0
  31. data/lib/antlr3/test/grammar.rb +371 -0
  32. data/lib/antlr3/token.rb +592 -0
  33. data/lib/antlr3/tree.rb +1415 -0
  34. data/lib/antlr3/tree/debug.rb +163 -0
  35. data/lib/antlr3/tree/visitor.rb +84 -0
  36. data/lib/antlr3/tree/wizard.rb +481 -0
  37. data/lib/antlr3/util.rb +149 -0
  38. data/lib/antlr3/version.rb +27 -0
  39. data/samples/ANTLRv3Grammar.g +621 -0
  40. data/samples/Cpp.g +749 -0
  41. data/templates/AST.stg +335 -0
  42. data/templates/ASTDbg.stg +40 -0
  43. data/templates/ASTParser.stg +153 -0
  44. data/templates/ASTTreeParser.stg +272 -0
  45. data/templates/Dbg.stg +192 -0
  46. data/templates/Ruby.stg +1514 -0
  47. data/test/functional/ast-output/auto-ast.rb +797 -0
  48. data/test/functional/ast-output/construction.rb +555 -0
  49. data/test/functional/ast-output/hetero-nodes.rb +753 -0
  50. data/test/functional/ast-output/rewrites.rb +1327 -0
  51. data/test/functional/ast-output/tree-rewrite.rb +1662 -0
  52. data/test/functional/debugging/debug-mode.rb +689 -0
  53. data/test/functional/debugging/profile-mode.rb +165 -0
  54. data/test/functional/debugging/rule-tracing.rb +74 -0
  55. data/test/functional/delegation/import.rb +379 -0
  56. data/test/functional/lexer/basic.rb +559 -0
  57. data/test/functional/lexer/filter-mode.rb +245 -0
  58. data/test/functional/lexer/nuances.rb +47 -0
  59. data/test/functional/lexer/properties.rb +104 -0
  60. data/test/functional/lexer/syn-pred.rb +32 -0
  61. data/test/functional/lexer/xml.rb +206 -0
  62. data/test/functional/main/main-scripts.rb +245 -0
  63. data/test/functional/parser/actions.rb +224 -0
  64. data/test/functional/parser/backtracking.rb +244 -0
  65. data/test/functional/parser/basic.rb +282 -0
  66. data/test/functional/parser/calc.rb +98 -0
  67. data/test/functional/parser/ll-star.rb +143 -0
  68. data/test/functional/parser/nuances.rb +165 -0
  69. data/test/functional/parser/predicates.rb +103 -0
  70. data/test/functional/parser/properties.rb +242 -0
  71. data/test/functional/parser/rule-methods.rb +132 -0
  72. data/test/functional/parser/scopes.rb +274 -0
  73. data/test/functional/token-rewrite/basic.rb +318 -0
  74. data/test/functional/token-rewrite/via-parser.rb +100 -0
  75. data/test/functional/tree-parser/basic.rb +750 -0
  76. data/test/unit/sample-input/file-stream-1 +2 -0
  77. data/test/unit/sample-input/teststreams.input2 +2 -0
  78. data/test/unit/test-dfa.rb +52 -0
  79. data/test/unit/test-exceptions.rb +44 -0
  80. data/test/unit/test-recognizers.rb +55 -0
  81. data/test/unit/test-scheme.rb +62 -0
  82. data/test/unit/test-streams.rb +459 -0
  83. data/test/unit/test-tree-wizard.rb +535 -0
  84. data/test/unit/test-trees.rb +854 -0
  85. metadata +205 -0
@@ -0,0 +1,559 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+ require 'antlr3/test/functional'
4
+
5
+ class LexerTest001 < ANTLR3::Test::Functional
6
+ inline_grammar(<<-'END')
7
+ lexer grammar Zero;
8
+ options {
9
+ language = Ruby;
10
+ }
11
+
12
+ @members { include ANTLR3::Test::RaiseErrors }
13
+
14
+ ZERO: '0';
15
+ END
16
+
17
+ example %(lexing '0') do
18
+ lexer = Zero::Lexer.new('0')
19
+
20
+ token = lexer.next_token
21
+ token.name.should == 'ZERO'
22
+
23
+ token = lexer.next_token
24
+ token.name.should == '<EOF>'
25
+ end
26
+
27
+ example %(iterating over tokens) do
28
+ lexer = Zero::Lexer.new('0')
29
+
30
+ token_types = lexer.map { |token| token.name }
31
+ token_types.should == %w(ZERO)
32
+ end
33
+
34
+ example "mismatched token" do
35
+ lexer = Zero::Lexer.new('1')
36
+
37
+ proc {
38
+ token = lexer.next_token
39
+ }.should raise_error(ANTLR3::Error::MismatchedToken) do |e|
40
+ e.expecting.should == '0'
41
+ e.unexpected_type.should == '1'
42
+ end
43
+ end
44
+ end
45
+
46
+ class LexerTest002 < ANTLR3::Test::Functional
47
+ inline_grammar(<<-'END')
48
+ lexer grammar Binary;
49
+ options {
50
+ language = Ruby;
51
+ }
52
+
53
+ @members { include ANTLR3::Test::RaiseErrors }
54
+
55
+ ZERO: '0';
56
+ ONE: '1';
57
+ END
58
+
59
+ example "lexing '01'" do
60
+ lexer = Binary::Lexer.new( '01' )
61
+
62
+ token = lexer.next_token
63
+ token.name.should == 'ZERO'
64
+
65
+ token = lexer.next_token
66
+ token.name.should == 'ONE'
67
+
68
+ token = lexer.next_token
69
+ token.name.should == '<EOF>'
70
+ end
71
+
72
+ example "no matching token rule" do
73
+ lexer = Binary::Lexer.new( '2' )
74
+
75
+ b = lambda { token = lexer.next_token }
76
+ b.should raise_error(ANTLR3::Error::NoViableAlternative) do |exc|
77
+ exc.unexpected_type.should == '2'
78
+ end
79
+ end
80
+
81
+ end
82
+
83
+ class LexerTest003 < ANTLR3::Test::Functional
84
+ inline_grammar(<<-'END')
85
+ lexer grammar BinaryFooze;
86
+ options {
87
+ language = Ruby;
88
+ }
89
+
90
+ @members { include ANTLR3::Test::RaiseErrors }
91
+
92
+ ZERO: '0';
93
+ ONE: '1';
94
+ FOOZE: 'fooze';
95
+ END
96
+
97
+ example "lexing '0fooze1'" do
98
+ lexer = BinaryFooze::Lexer.new( '0fooze1' )
99
+
100
+ token = lexer.next_token
101
+ token.name.should == 'ZERO'
102
+
103
+ token = lexer.next_token
104
+ token.name.should == 'FOOZE'
105
+
106
+ token = lexer.next_token
107
+ token.name.should == 'ONE'
108
+
109
+ token = lexer.next_token
110
+ token.name.should == '<EOF>'
111
+ end
112
+
113
+ example "no token match" do
114
+ lexer = BinaryFooze::Lexer.new( '2' )
115
+
116
+ proc { lexer.next_token }.
117
+ should raise_error(ANTLR3::Error::NoViableAlternative) do |exc|
118
+ exc.unexpected_type.should == '2'
119
+ end
120
+ end
121
+ end
122
+
123
+
124
+ class LexerTest004 < ANTLR3::Test::Functional
125
+ inline_grammar(<<-'END')
126
+ lexer grammar FooStar;
127
+ options {
128
+ language = Ruby;
129
+ }
130
+
131
+ @members { include ANTLR3::Test::RaiseErrors }
132
+
133
+ FOO: 'f' 'o'*;
134
+ END
135
+
136
+ example "lexing 'ffofoofooo'" do
137
+ lexer = FooStar::Lexer.new( 'ffofoofooo' )
138
+
139
+ token = lexer.next_token
140
+ token.name.should == 'FOO'
141
+ token.start.should == 0
142
+ token.stop.should == 0
143
+ token.text.should == 'f'
144
+
145
+ token = lexer.next_token
146
+ token.name.should == 'FOO'
147
+ token.text.should == 'fo'
148
+ token.start.should == 1
149
+ token.stop.should == 2
150
+
151
+ token = lexer.next_token
152
+ token.name.should == 'FOO'
153
+ token.start.should == 3
154
+ token.stop.should == 5
155
+ token.text.should == 'foo'
156
+
157
+ token = lexer.next_token
158
+ token.name.should == 'FOO'
159
+ token.start.should == 6
160
+ token.stop.should == 9
161
+ token.text.should == 'fooo'
162
+
163
+ token = lexer.next_token
164
+ token.name.should == '<EOF>'
165
+ end
166
+
167
+ example "mismatched token" do
168
+ lexer = FooStar::Lexer.new( '2' )
169
+
170
+ proc { lexer.next_token }.
171
+ should raise_error(ANTLR3::Error::MismatchedToken) do |exc|
172
+ exc.expecting.should == 'f'
173
+ exc.unexpected_type.should == '2'
174
+ end
175
+ end
176
+ end
177
+
178
+ class LexerTest005 < ANTLR3::Test::Functional
179
+ inline_grammar(<<-'END')
180
+ lexer grammar FooPlus;
181
+ options {
182
+ language = Ruby;
183
+ }
184
+
185
+ @members { include ANTLR3::Test::RaiseErrors }
186
+
187
+ FOO: 'f' 'o'+;
188
+ END
189
+
190
+ example "lexing 'fofoofooo'" do
191
+ lexer = FooPlus::Lexer.new( 'fofoofooo' )
192
+
193
+ token = lexer.next_token
194
+ token.name.should == 'FOO'
195
+ token.start.should == 0
196
+ token.stop.should == 1
197
+ token.text.should == 'fo'
198
+
199
+ token = lexer.next_token
200
+ token.name.should == 'FOO'
201
+ token.text.should == 'foo'
202
+ token.start.should == 2
203
+ token.stop.should == 4
204
+
205
+ token = lexer.next_token
206
+ token.name.should == 'FOO'
207
+ token.start.should == 5
208
+ token.stop.should == 8
209
+ token.text.should == 'fooo'
210
+
211
+ token = lexer.next_token
212
+ token.name.should == '<EOF>'
213
+ end
214
+
215
+ example "mismatched token" do
216
+ lexer = FooPlus::Lexer.new( '2' )
217
+
218
+ proc { lexer.next_token }.
219
+ should raise_error(ANTLR3::Error::MismatchedToken) do |exc|
220
+ exc.expecting.should == 'f'
221
+ exc.unexpected_type.should == '2'
222
+ end
223
+ end
224
+
225
+ example "early exit" do
226
+ lexer = FooPlus::Lexer.new( 'f' )
227
+
228
+ proc { token = lexer.next_token }.
229
+ should raise_error(ANTLR3::Error::EarlyExit) { |exc|
230
+ exc.unexpected_type.should == ANTLR3::Constants::EOF
231
+ }
232
+ end
233
+
234
+ end
235
+
236
+ class LexerTest006 < ANTLR3::Test::Functional
237
+ inline_grammar(<<-'END')
238
+ lexer grammar FoaStar;
239
+ options {
240
+ language = Ruby;
241
+ }
242
+
243
+ @members { include ANTLR3::Test::RaiseErrors }
244
+
245
+ FOO: 'f' ('o' | 'a')*;
246
+ END
247
+
248
+ example "lexing 'fofaaooa'" do
249
+ lexer = FoaStar::Lexer.new( 'fofaaooa' )
250
+
251
+ token = lexer.next_token
252
+ token.name.should == 'FOO'
253
+ token.start.should == 0
254
+ token.stop.should == 1
255
+ token.text.should == 'fo'
256
+
257
+ token = lexer.next_token
258
+ token.name.should == 'FOO'
259
+ token.text.should == 'faaooa'
260
+ token.start.should == 2
261
+ token.stop.should == 7
262
+
263
+ token = lexer.next_token
264
+ token.name.should == '<EOF>'
265
+ end
266
+
267
+ example "mismatched token" do
268
+ lexer = FoaStar::Lexer.new( 'fofoaooaoa2' )
269
+
270
+ lexer.next_token
271
+ lexer.next_token
272
+ proc { lexer.next_token }.
273
+ should raise_error(ANTLR3::Error::MismatchedToken) do |exc|
274
+ exc.expecting.should == 'f'
275
+ exc.unexpected_type.should == '2'
276
+ exc.column.should == 10
277
+ exc.line.should == 1
278
+ end
279
+ end
280
+ end
281
+
282
+ class LexerTest007 < ANTLR3::Test::Functional
283
+ inline_grammar(<<-'END')
284
+ lexer grammar Foab;
285
+ options {
286
+ language = Ruby;
287
+ }
288
+
289
+ @members { include ANTLR3::Test::RaiseErrors }
290
+
291
+ FOO: 'f' ('o' | 'a' 'b'+)*;
292
+ END
293
+
294
+ example "lexing 'fofababbooabb'" do
295
+ lexer = Foab::Lexer.new( 'fofababbooabb' )
296
+
297
+ token = lexer.next_token
298
+ token.name.should == 'FOO'
299
+ token.start.should == 0
300
+ token.stop.should == 1
301
+ token.text.should == 'fo'
302
+
303
+ token = lexer.next_token
304
+ token.name.should == 'FOO'
305
+ token.start.should == 2
306
+ token.stop.should == 12
307
+ token.text.should == 'fababbooabb'
308
+
309
+ token = lexer.next_token
310
+ token.name.should == '<EOF>'
311
+ end
312
+
313
+ example "early exit" do
314
+ lexer = Foab::Lexer.new( 'foaboao' )
315
+
316
+ proc { lexer.next_token }.
317
+ should raise_error(ANTLR3::Error::EarlyExit) do |exc|
318
+ exc.unexpected_type.should == 'o'
319
+ exc.column.should == 6
320
+ exc.line.should == 1
321
+ end
322
+ end
323
+ end
324
+
325
+ class LexerTest008 < ANTLR3::Test::Functional
326
+ inline_grammar(<<-'END')
327
+ lexer grammar Fa;
328
+ options {
329
+ language = Ruby;
330
+ }
331
+
332
+ @members { include ANTLR3::Test::RaiseErrors }
333
+
334
+ FOO: 'f' 'a'?;
335
+ END
336
+
337
+ example "lexing 'ffaf'" do
338
+ lexer = Fa::Lexer.new( 'ffaf' )
339
+
340
+ token = lexer.next_token
341
+ token.name.should == 'FOO'
342
+ token.start.should == 0
343
+ token.stop.should == 0
344
+ token.text.should == 'f'
345
+
346
+ token = lexer.next_token
347
+ token.name.should == 'FOO'
348
+ token.start.should == 1
349
+ token.stop.should == 2
350
+ token.text.should == 'fa'
351
+
352
+ token = lexer.next_token
353
+ token.name.should == 'FOO'
354
+ token.start.should == 3
355
+ token.stop.should == 3
356
+ token.text.should == 'f'
357
+
358
+ token = lexer.next_token
359
+ token.name.should == '<EOF>'
360
+ end
361
+
362
+ example "mismatched token" do
363
+ lexer = Fa::Lexer.new( 'fafb' )
364
+
365
+ lexer.next_token
366
+ lexer.next_token
367
+ proc { lexer.next_token }.
368
+ should raise_error(ANTLR3::Error::MismatchedToken) do |exc|
369
+ exc.unexpected_type.should == 'b'
370
+ exc.column.should == 3
371
+ exc.line.should == 1
372
+ end
373
+ end
374
+ end
375
+
376
+
377
+ class LexerTest009 < ANTLR3::Test::Functional
378
+ inline_grammar(<<-'END')
379
+ lexer grammar Digit;
380
+ options {
381
+ language = Ruby;
382
+ }
383
+
384
+ @members { include ANTLR3::Test::RaiseErrors }
385
+
386
+ DIGIT: '0' .. '9';
387
+ END
388
+
389
+ example "lexing '085'" do
390
+ lexer = Digit::Lexer.new( '085' )
391
+
392
+ token = lexer.next_token
393
+ token.name.should == 'DIGIT'
394
+ token.start.should == 0
395
+ token.stop.should == 0
396
+ token.text.should == '0'
397
+
398
+ token = lexer.next_token
399
+ token.name.should == 'DIGIT'
400
+ token.start.should == 1
401
+ token.stop.should == 1
402
+ token.text.should == '8'
403
+
404
+ token = lexer.next_token
405
+ token.name.should == 'DIGIT'
406
+ token.start.should == 2
407
+ token.stop.should == 2
408
+ token.text.should == '5'
409
+
410
+ token = lexer.next_token
411
+ token.name.should == '<EOF>'
412
+ end
413
+
414
+ example "mismatched range" do
415
+ lexer = Digit::Lexer.new( '2a' )
416
+
417
+ lexer.next_token
418
+ proc { lexer.next_token }.
419
+ should raise_error(ANTLR3::Error::MismatchedRange) do |exc|
420
+ exc.min.should == '0'
421
+ exc.max.should == '9'
422
+ exc.unexpected_type.should == 'a'
423
+ exc.column.should == 1
424
+ exc.line.should == 1
425
+ end
426
+ end
427
+ end
428
+
429
+ class LexerTest010 < ANTLR3::Test::Functional
430
+ inline_grammar(<<-'END')
431
+ lexer grammar IDsAndSpaces;
432
+ options {
433
+ language = Ruby;
434
+ }
435
+
436
+ @members { include ANTLR3::Test::RaiseErrors }
437
+
438
+ IDENTIFIER: ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*;
439
+ WS: (' ' | '\n')+;
440
+ END
441
+
442
+ example "lexing 'foobar _Ab98 \n A12sdf'" do
443
+ lexer = IDsAndSpaces::Lexer.new( "foobar _Ab98 \n A12sdf" )
444
+
445
+ token = lexer.next_token
446
+ token.name.should == 'IDENTIFIER'
447
+ token.start.should == 0
448
+ token.stop.should == 5
449
+ token.text.should == 'foobar'
450
+
451
+ token = lexer.next_token
452
+ token.name.should == 'WS'
453
+ token.start.should == 6
454
+ token.stop.should == 6
455
+ token.text.should == ' '
456
+
457
+ token = lexer.next_token
458
+ token.name.should == 'IDENTIFIER'
459
+ token.start.should == 7
460
+ token.stop.should == 11
461
+ token.text.should == '_Ab98'
462
+
463
+ token = lexer.next_token
464
+ token.name.should == 'WS'
465
+ token.start.should == 12
466
+ token.stop.should == 14
467
+ token.text.should == " \n "
468
+
469
+ token = lexer.next_token
470
+ token.name.should == 'IDENTIFIER'
471
+ token.start.should == 15
472
+ token.stop.should == 20
473
+ token.text.should == 'A12sdf'
474
+
475
+ token = lexer.next_token
476
+ token.name.should == '<EOF>'
477
+ end
478
+
479
+ example "contains characters without a matching token rule" do
480
+ lexer = IDsAndSpaces::Lexer.new( 'a-b' )
481
+
482
+ lexer.next_token
483
+ proc { lexer.next_token }.
484
+ should raise_error(ANTLR3::Error::NoViableAlternative) do |exc|
485
+ exc.unexpected_type.should == '-'
486
+ exc.column.should == 1
487
+ exc.line.should == 1
488
+ end
489
+ end
490
+ end
491
+
492
+ class LexerTest011 < ANTLR3::Test::Functional
493
+ inline_grammar(<<-'END')
494
+ lexer grammar IDsWithAction;
495
+ options {language = Ruby;}
496
+
497
+ @members { include ANTLR3::Test::RaiseErrors }
498
+
499
+ IDENTIFIER:
500
+ ('a'..'z'|'A'..'Z'|'_')
501
+ ('a'..'z'
502
+ |'A'..'Z'
503
+ |'0'..'9'
504
+ |'_' { \$action_var = '_' }
505
+ )*
506
+ ;
507
+
508
+ WS: (' ' | '\n')+;
509
+ END
510
+
511
+ example "lexing 'foobar _Ab98 \n A12sdf'" do
512
+ lexer = IDsWithAction::Lexer.new( "foobar _Ab98 \n A12sdf" )
513
+
514
+ token = lexer.next_token
515
+ token.name.should == 'IDENTIFIER'
516
+ token.start.should == 0
517
+ token.stop.should == 5
518
+ token.text.should == 'foobar'
519
+
520
+ token = lexer.next_token
521
+ token.name.should == 'WS'
522
+ token.start.should == 6
523
+ token.stop.should == 6
524
+ token.text.should == ' '
525
+
526
+ token = lexer.next_token
527
+ token.name.should == 'IDENTIFIER'
528
+ token.start.should == 7
529
+ token.stop.should == 11
530
+ token.text.should == '_Ab98'
531
+
532
+ token = lexer.next_token
533
+ token.name.should == 'WS'
534
+ token.start.should == 12
535
+ token.stop.should == 14
536
+ token.text.should == " \n "
537
+
538
+ token = lexer.next_token
539
+ token.name.should == 'IDENTIFIER'
540
+ token.start.should == 15
541
+ token.stop.should == 20
542
+ token.text.should == 'A12sdf'
543
+
544
+ token = lexer.next_token
545
+ token.name.should == '<EOF>'
546
+ end
547
+
548
+ example "contains characters without a matching token" do
549
+ lexer = IDsWithAction::Lexer.new( 'a-b' )
550
+
551
+ lexer.next_token
552
+ proc { lexer.next_token }.
553
+ should raise_error(ANTLR3::Error::NoViableAlternative) do |exc|
554
+ exc.unexpected_type.should == '-'
555
+ exc.column.should == 1
556
+ exc.line.should == 1
557
+ end
558
+ end
559
+ end