antlr3 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/ANTLR-LICENSE.txt +26 -0
  2. data/History.txt +66 -0
  3. data/README.txt +139 -0
  4. data/bin/antlr4ruby +33 -0
  5. data/java/RubyTarget.java +524 -0
  6. data/java/antlr-full-3.2.1.jar +0 -0
  7. data/lib/antlr3.rb +176 -0
  8. data/lib/antlr3/constants.rb +88 -0
  9. data/lib/antlr3/debug.rb +701 -0
  10. data/lib/antlr3/debug/event-hub.rb +210 -0
  11. data/lib/antlr3/debug/record-event-listener.rb +25 -0
  12. data/lib/antlr3/debug/rule-tracer.rb +55 -0
  13. data/lib/antlr3/debug/socket.rb +360 -0
  14. data/lib/antlr3/debug/trace-event-listener.rb +92 -0
  15. data/lib/antlr3/dfa.rb +247 -0
  16. data/lib/antlr3/dot.rb +174 -0
  17. data/lib/antlr3/error.rb +657 -0
  18. data/lib/antlr3/main.rb +561 -0
  19. data/lib/antlr3/modes/ast-builder.rb +41 -0
  20. data/lib/antlr3/modes/filter.rb +56 -0
  21. data/lib/antlr3/profile.rb +322 -0
  22. data/lib/antlr3/recognizers.rb +1280 -0
  23. data/lib/antlr3/streams.rb +985 -0
  24. data/lib/antlr3/streams/interactive.rb +91 -0
  25. data/lib/antlr3/streams/rewrite.rb +412 -0
  26. data/lib/antlr3/test/call-stack.rb +57 -0
  27. data/lib/antlr3/test/config.rb +23 -0
  28. data/lib/antlr3/test/core-extensions.rb +269 -0
  29. data/lib/antlr3/test/diff.rb +165 -0
  30. data/lib/antlr3/test/functional.rb +207 -0
  31. data/lib/antlr3/test/grammar.rb +371 -0
  32. data/lib/antlr3/token.rb +592 -0
  33. data/lib/antlr3/tree.rb +1415 -0
  34. data/lib/antlr3/tree/debug.rb +163 -0
  35. data/lib/antlr3/tree/visitor.rb +84 -0
  36. data/lib/antlr3/tree/wizard.rb +481 -0
  37. data/lib/antlr3/util.rb +149 -0
  38. data/lib/antlr3/version.rb +27 -0
  39. data/samples/ANTLRv3Grammar.g +621 -0
  40. data/samples/Cpp.g +749 -0
  41. data/templates/AST.stg +335 -0
  42. data/templates/ASTDbg.stg +40 -0
  43. data/templates/ASTParser.stg +153 -0
  44. data/templates/ASTTreeParser.stg +272 -0
  45. data/templates/Dbg.stg +192 -0
  46. data/templates/Ruby.stg +1514 -0
  47. data/test/functional/ast-output/auto-ast.rb +797 -0
  48. data/test/functional/ast-output/construction.rb +555 -0
  49. data/test/functional/ast-output/hetero-nodes.rb +753 -0
  50. data/test/functional/ast-output/rewrites.rb +1327 -0
  51. data/test/functional/ast-output/tree-rewrite.rb +1662 -0
  52. data/test/functional/debugging/debug-mode.rb +689 -0
  53. data/test/functional/debugging/profile-mode.rb +165 -0
  54. data/test/functional/debugging/rule-tracing.rb +74 -0
  55. data/test/functional/delegation/import.rb +379 -0
  56. data/test/functional/lexer/basic.rb +559 -0
  57. data/test/functional/lexer/filter-mode.rb +245 -0
  58. data/test/functional/lexer/nuances.rb +47 -0
  59. data/test/functional/lexer/properties.rb +104 -0
  60. data/test/functional/lexer/syn-pred.rb +32 -0
  61. data/test/functional/lexer/xml.rb +206 -0
  62. data/test/functional/main/main-scripts.rb +245 -0
  63. data/test/functional/parser/actions.rb +224 -0
  64. data/test/functional/parser/backtracking.rb +244 -0
  65. data/test/functional/parser/basic.rb +282 -0
  66. data/test/functional/parser/calc.rb +98 -0
  67. data/test/functional/parser/ll-star.rb +143 -0
  68. data/test/functional/parser/nuances.rb +165 -0
  69. data/test/functional/parser/predicates.rb +103 -0
  70. data/test/functional/parser/properties.rb +242 -0
  71. data/test/functional/parser/rule-methods.rb +132 -0
  72. data/test/functional/parser/scopes.rb +274 -0
  73. data/test/functional/token-rewrite/basic.rb +318 -0
  74. data/test/functional/token-rewrite/via-parser.rb +100 -0
  75. data/test/functional/tree-parser/basic.rb +750 -0
  76. data/test/unit/sample-input/file-stream-1 +2 -0
  77. data/test/unit/sample-input/teststreams.input2 +2 -0
  78. data/test/unit/test-dfa.rb +52 -0
  79. data/test/unit/test-exceptions.rb +44 -0
  80. data/test/unit/test-recognizers.rb +55 -0
  81. data/test/unit/test-scheme.rb +62 -0
  82. data/test/unit/test-streams.rb +459 -0
  83. data/test/unit/test-tree-wizard.rb +535 -0
  84. data/test/unit/test-trees.rb +854 -0
  85. metadata +205 -0
@@ -0,0 +1,245 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3'
5
+ require 'antlr3/test/functional'
6
+
7
+ ENV.delete( 'RUBYOPT' )
8
+ ENV['RUBYLIB'] = ANTLR3.library_path
9
+
10
+ class TestMainUtility < ANTLR3::Test::Functional
11
+
12
+ example 'overriding the built-in script action using the @main named-action' do
13
+ grammar = inline_grammar(<<-'END')
14
+ lexer grammar MainOverride;
15
+ options { language = Ruby; }
16
+
17
+ @main {
18
+ raise( "the main block ran" )
19
+ }
20
+
21
+ ID: ('a'..'z' | '\u00c0'..'\u00ff')+;
22
+ WS: ' '+ { $channel = HIDDEN; };
23
+ END
24
+
25
+ # when this grammar is compiled and the resulting ruby files
26
+ # are loaded as a library, the custom @main block
27
+ # should not be executed
28
+ proc { compile_and_load( grammar ) }.should_not raise_error
29
+
30
+ # this assertion verifies that the main region is executed
31
+ # when the parser script is run directly
32
+ lexer_script = grammar.target_files.first
33
+ out = `ruby #{lexer_script} 2>&1`.chomp
34
+ out.should =~ /the main block ran/
35
+ end
36
+
37
+ example 'using Lexer.main() to run the built-in lexer utility script on a source file' do
38
+ input_path = local_path('input.txt')
39
+ open(input_path, 'w') { |f| f.write("yada yada") }
40
+
41
+ compile_and_load inline_grammar(<<-'END')
42
+ lexer grammar LexerMainWithSourceFile;
43
+ options { language = Ruby; }
44
+
45
+ ID: 'a'..'z'+;
46
+ WS: ' '+ { $channel = HIDDEN; };
47
+ END
48
+
49
+ begin
50
+ output = StringIO.new
51
+ input = File.open(input_path)
52
+ LexerMainWithSourceFile::Lexer.main([], :input => input, :output => output)
53
+
54
+ out_lines = output.string.split(/\n/)
55
+ out_lines.should have(3).things
56
+ ensure
57
+ File.delete(input_path)
58
+ end
59
+ end
60
+
61
+ example 'using Lexer.main to run the built-in lexer utility script on input from $stdin' do
62
+ input = StringIO.new("yada yada") # <- used to simulate $stdin
63
+ output = StringIO.new
64
+
65
+ compile_and_load inline_grammar(<<-'END')
66
+ lexer grammar LexerMainFromStdIO;
67
+ options { language = Ruby; }
68
+
69
+ ID: 'a'..'z'+;
70
+ WS: ' '+ { $channel = HIDDEN; };
71
+ END
72
+
73
+ LexerMainFromStdIO::Lexer.main([], :input => input, :output => output)
74
+ lines = output.string.split(/\n/)
75
+ lines.should have(3).things
76
+ end
77
+
78
+ example 'using Parser.main to run the built-in parser script utility with a combo grammar' do
79
+ compile_and_load inline_grammar(<<-'END')
80
+ grammar MainForCombined;
81
+ options { language = Ruby; }
82
+ r returns [res]: (ID)+ EOF { $res = $text; };
83
+
84
+ ID: 'a'..'z'+;
85
+ WS: ' '+ { $channel = HIDDEN; };
86
+ END
87
+
88
+ output = StringIO.new
89
+ input = StringIO.new('yada yada')
90
+
91
+ MainForCombined::Parser.main(
92
+ %w(--rule r --lexer-name MainForCombined::Lexer),
93
+ :input => input, :output => output)
94
+ lines = output.string.split("\n")
95
+ lines.should have(4).things
96
+ end
97
+
98
+ example 'using built-in main to inspect AST constructed by an AST-building parser' do
99
+ compile_and_load inline_grammar(<<-'END')
100
+ grammar ASTParserMain;
101
+ options {
102
+ language = Ruby;
103
+ output = AST;
104
+ }
105
+ r: ID OP^ ID EOF!;
106
+
107
+ ID: 'a'..'z'+;
108
+ OP: '+';
109
+ WS: ' '+ { $channel = HIDDEN; };
110
+ END
111
+
112
+ output = StringIO.new
113
+ input = StringIO.new 'yada + yada'
114
+ ASTParserMain::Parser.main(
115
+ %w(--rule r --lexer-name ASTParserMain::Lexer),
116
+ :input => input, :output => output)
117
+ output = output.string.strip
118
+ output.should == "(+ yada yada)"
119
+ end
120
+
121
+ example "using a tree parser's built-in main" do
122
+ compile_and_load inline_grammar(<<-'END')
123
+ grammar TreeMain;
124
+ options {
125
+ language = Ruby;
126
+ output = AST;
127
+ }
128
+
129
+ r: ID OP^ ID EOF!;
130
+
131
+ ID: 'a'..'z'+;
132
+ OP: '+';
133
+ WS: ' '+ { $channel = HIDDEN; };
134
+ END
135
+ compile_and_load inline_grammar(<<-'END')
136
+ tree grammar TreeMainWalker;
137
+ options {
138
+ language=Ruby;
139
+ ASTLabelType=CommonTree;
140
+ tokenVocab=TreeMain;
141
+ }
142
+ r returns [res]: ^(OP a=ID b=ID)
143
+ { $res = "\%s \%s \%s" \% [$a.text, $OP.text, $b.text] }
144
+ ;
145
+ END
146
+
147
+ output = StringIO.new
148
+ input = StringIO.new 'a+b'
149
+
150
+ TreeMainWalker::TreeParser.main(
151
+ %w(--rule r --parser-name TreeMain::Parser
152
+ --parser-rule r --lexer-name TreeMain::Lexer),
153
+ :input => input, :output => output)
154
+ output = output.string.strip
155
+ output.should == '"a + b"'
156
+ end
157
+
158
+ example "using a tree parser's built-in main to inspect AST rewrite output" do
159
+ compile_and_load inline_grammar(<<-'END')
160
+ grammar TreeRewriteMain;
161
+ options {
162
+ language = Ruby;
163
+ output = AST;
164
+ }
165
+
166
+ r: ID OP^ ID EOF!;
167
+
168
+ ID: 'a'..'z'+;
169
+ OP: '+';
170
+ WS: ' '+ { $channel = HIDDEN; };
171
+ END
172
+ compile_and_load inline_grammar(<<-'END')
173
+ tree grammar TreeRewriteMainWalker;
174
+ options {
175
+ language=Ruby;
176
+ ASTLabelType=CommonTree;
177
+ tokenVocab=TreeRewriteMain;
178
+ output=AST;
179
+ }
180
+ tokens { ARG; }
181
+ r: ^(OP a=ID b=ID) -> ^(OP ^(ARG ID) ^(ARG ID));
182
+ END
183
+
184
+ output = StringIO.new
185
+ input = StringIO.new 'a+b'
186
+ TreeRewriteMainWalker::TreeParser.main(
187
+ %w(--rule r --parser-name TreeRewriteMain::Parser
188
+ --parser-rule r --lexer-name TreeRewriteMain::Lexer),
189
+ :input => input, :output => output
190
+ )
191
+
192
+ output = output.string.strip
193
+ output.should == '(+ (ARG a) (ARG b))'
194
+ end
195
+
196
+ example 'using built-in main with a delegating grammar' do
197
+ inline_grammar(<<-'END')
198
+ parser grammar MainSlave;
199
+ options { language=Ruby; }
200
+ a : B;
201
+ END
202
+ master = inline_grammar(<<-'END')
203
+ grammar MainMaster;
204
+ options { language=Ruby; }
205
+ import MainSlave;
206
+ s returns [res]: a { $res = $a.text };
207
+ B : 'b' ; // defines B from inherited token space
208
+ WS : (' '|'\n') {skip} ;
209
+ END
210
+ master.compile
211
+ for file in master.target_files
212
+ require(file)
213
+ end
214
+
215
+ output = StringIO.new
216
+ input = StringIO.new 'b'
217
+
218
+ MainMaster::Parser.main(
219
+ %w(--rule s --lexer-name MainMaster::Lexer),
220
+ :input => input, :output => output)
221
+ output = output.string.strip
222
+ output.should == 'b'.inspect
223
+ end
224
+
225
+ #test :LexerEncoding do
226
+ # broken!("Non-ASCII encodings have not been implemented yet")
227
+ # grammar = inline_grammar(<<-'END')
228
+ # lexer grammar T3;
229
+ # options {
230
+ # language = Ruby;
231
+ # }
232
+ #
233
+ # ID: ('a'..'z' | '\u00c0'..'\u00ff')+;
234
+ # WS: ' '+ { $channel = HIDDEN; };
235
+ # END
236
+ # compile grammar
237
+ # input = StringIO.new("föö bär")
238
+ # output = StringIO.new('')
239
+ # lexer_class.main(%w(--encoding utf-8), :input => input, :output => output)
240
+ # puts output.string
241
+ # lines = output.string.split(/\n/)
242
+ # lines.should have(3).things
243
+ #end
244
+
245
+ end
@@ -0,0 +1,224 @@
1
+ #!/usr/bin/ruby
2
+ # encoding: utf-8
3
+
4
+ require 'antlr3/test/functional'
5
+
6
+ class TestActions1 < ANTLR3::Test::Functional
7
+ inline_grammar(<<-'END')
8
+ grammar ParserActions;
9
+ options { language = Ruby; }
10
+
11
+ declaration returns [name]
12
+ : functionHeader ';'
13
+ { $name = $functionHeader.name }
14
+ ;
15
+
16
+ functionHeader returns [name]
17
+ : type id=ID
18
+ { $name = $id.text }
19
+ ;
20
+
21
+ type
22
+ : 'int'
23
+ | 'char'
24
+ | 'void'
25
+ ;
26
+
27
+ ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
28
+ ;
29
+
30
+ WS : ( ' '
31
+ | '\t'
32
+ | '\r'
33
+ | '\n'
34
+ )+
35
+ {$channel=HIDDEN}
36
+ ;
37
+ END
38
+
39
+ example "parser action execution" do
40
+ lexer = ParserActions::Lexer.new "int foo;"
41
+ parser = ParserActions::Parser.new lexer
42
+
43
+ parser.declaration.should == 'foo'
44
+ end
45
+
46
+ end
47
+
48
+
49
+ class TestActions2 < ANTLR3::Test::Functional
50
+
51
+ inline_grammar(<<-'END')
52
+ grammar AllKindsOfActions;
53
+ options { language = Ruby; }
54
+
55
+ @parser::members {
56
+ include ANTLR3::Test::CaptureOutput
57
+ }
58
+
59
+ @lexer::members {
60
+ include ANTLR3::Test::CaptureOutput
61
+ }
62
+ @lexer::init { @foobar = 'attribute' }
63
+
64
+ prog
65
+ @init { say('init') }
66
+ @after { say('after') }
67
+ : IDENTIFIER EOF
68
+ ;
69
+ catch [ RecognitionError => exc ] {
70
+ say('catch')
71
+ raise
72
+ }
73
+ finally { say('finally') }
74
+
75
+
76
+ IDENTIFIER
77
+ : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
78
+ {
79
+ # a comment
80
+ say('action')
81
+ say('\%p \%p \%p \%p \%p \%p \%p \%p' \% [$text, $type, $line, $pos, $index, $channel, $start, $stop])
82
+ say(@foobar)
83
+ }
84
+ ;
85
+
86
+ WS: (' ' | '\n')+;
87
+ END
88
+
89
+
90
+ example "execution of special named actions" do
91
+ lexer = AllKindsOfActions::Lexer.new( "foobar _Ab98 \n A12sdf" )
92
+ parser = AllKindsOfActions::Parser.new lexer
93
+ parser.prog
94
+
95
+ parser.output.should == <<-END.fixed_indent(0)
96
+ init
97
+ after
98
+ finally
99
+ END
100
+
101
+ lexer.output.should == <<-END.fixed_indent(0)
102
+ action
103
+ "foobar" 4 1 0 -1 :default 0 5
104
+ attribute
105
+ action
106
+ "_Ab98" 4 1 7 -1 :default 7 11
107
+ attribute
108
+ action
109
+ "A12sdf" 4 2 1 -1 :default 15 20
110
+ attribute
111
+ END
112
+ end
113
+ end
114
+
115
+ class TestFinally < ANTLR3::Test::Functional
116
+
117
+ inline_grammar(<<-'END')
118
+ grammar Finally;
119
+
120
+ options {
121
+ language = Ruby;
122
+ }
123
+
124
+ prog returns [events]
125
+ @init {events = []}
126
+ @after {events << 'after'}
127
+ : ID {raise RuntimeError}
128
+ ;
129
+ catch [RuntimeError] {events << 'catch'}
130
+ finally { events << 'finally'}
131
+
132
+ ID : ('a'..'z')+
133
+ ;
134
+
135
+ WS : (' '|'\n'|'\r')+ {$channel=HIDDEN}
136
+ ;
137
+ END
138
+
139
+
140
+ example "order of catch and ensure clauses" do
141
+ lexer = Finally::Lexer.new( 'foobar' )
142
+ parser = Finally::Parser.new lexer
143
+ parser.prog.should == %w(catch finally)
144
+ end
145
+
146
+ end
147
+
148
+
149
+ class TestActionScopes < ANTLR3::Test::Functional
150
+
151
+ inline_grammar(<<-'END')
152
+ grammar SpecialActionScopes;
153
+ options { language=Ruby; }
154
+
155
+ @header {
156
+ \$header_location = __LINE__
157
+ \$header_context = self
158
+ }
159
+
160
+ @footer {
161
+ \$footer_location = __LINE__
162
+ \$footer_context = self
163
+ }
164
+
165
+
166
+ @module::head {
167
+ \$module_head_location = __LINE__
168
+ \$module_head_context = self
169
+
170
+ class << self
171
+ attr_accessor :head_var
172
+ end
173
+ }
174
+
175
+ @module::foot {
176
+ \$module_foot_location = __LINE__
177
+ \$module_foot_context = self
178
+
179
+ FOOT_CONST = 1
180
+ }
181
+
182
+ @token::scheme {
183
+ \$token_scheme_location = __LINE__
184
+ \$token_scheme_context = self
185
+
186
+ SCHEME_CONST = 1
187
+ }
188
+
189
+ @token::members {
190
+ \$token_members_location = __LINE__
191
+ \$token_members_context = self
192
+
193
+ def value
194
+ text.to_i
195
+ end
196
+ }
197
+
198
+ @members {
199
+ \$members_location = __LINE__
200
+ }
201
+
202
+ nums returns [ds]: digs+=DIGIT+
203
+ { $ds = $digs.map { |t| t.value } };
204
+
205
+ DIGIT: ('0'..'9')+;
206
+ WS: (' ' | '\t' | '\n' | '\r' | '\f')+ { $channel=HIDDEN; };
207
+ END
208
+
209
+ example 'verifying action scope behavior' do
210
+ lexer = SpecialActionScopes::Lexer.new("10 20 30 40 50")
211
+ parser = SpecialActionScopes::Parser.new lexer
212
+ parser.nums.should == [10, 20, 30, 40, 50]
213
+ end
214
+
215
+ example 'special action scope locations' do
216
+ $header_location.should be < $module_head_location
217
+ $module_head_location.should be < $token_scheme_location
218
+ $token_scheme_location.should be < $token_members_location
219
+ $token_members_location.should be < $members_location
220
+ $members_location.should be < $module_foot_location
221
+ $module_foot_location.should be < $footer_location
222
+ end
223
+
224
+ end