antlr3 1.6.3 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Manifest.txt +3 -3
- data/java/RubyTarget.java +374 -364
- data/java/antlr-full-3.2.1.jar +0 -0
- data/lib/antlr3/error.rb +1 -1
- data/lib/antlr3/main.rb +2 -2
- data/lib/antlr3/recognizers.rb +8 -16
- data/lib/antlr3/streams.rb +51 -0
- data/lib/antlr3/task.rb +1 -0
- data/lib/antlr3/template.rb +5 -2
- data/lib/antlr3/template/{group-lexer.rb → group-file-lexer.rb} +57 -57
- data/lib/antlr3/template/{group-parser.rb → group-file-parser.rb} +110 -110
- data/lib/antlr3/template/group-file.rb +20 -0
- data/lib/antlr3/tree.rb +16 -14
- data/lib/antlr3/tree/debug.rb +7 -7
- data/lib/antlr3/tree/wizard.rb +3 -3
- data/lib/antlr3/version.rb +2 -2
- data/samples/ANTLRv3Grammar.g +0 -9
- data/templates/AST.stg +52 -58
- data/templates/ASTDbg.stg +13 -14
- data/templates/ASTParser.stg +16 -25
- data/templates/ASTTreeParser.stg +34 -64
- data/templates/Dbg.stg +6 -6
- data/templates/Ruby.stg +159 -191
- data/test/functional/debugging/debug-mode.rb +0 -1
- data/test/functional/parser/actions.rb +15 -1
- data/test/unit/test-trees.rb +7 -7
- metadata +7 -6
data/Manifest.txt
CHANGED
@@ -24,8 +24,9 @@ lib/antlr3/template.rb
|
|
24
24
|
lib/antlr3/dfa.rb
|
25
25
|
lib/antlr3/profile.rb
|
26
26
|
lib/antlr3/template/parameter.rb
|
27
|
-
lib/antlr3/template/group-
|
28
|
-
lib/antlr3/template/group-parser.rb
|
27
|
+
lib/antlr3/template/group-file.rb
|
28
|
+
lib/antlr3/template/group-file-parser.rb
|
29
|
+
lib/antlr3/template/group-file-lexer.rb
|
29
30
|
lib/antlr3/tree/visitor.rb
|
30
31
|
lib/antlr3/tree/debug.rb
|
31
32
|
lib/antlr3/tree/wizard.rb
|
@@ -87,7 +88,6 @@ samples/ANTLRv3Grammar.g
|
|
87
88
|
README.txt
|
88
89
|
ANTLR-LICENSE.txt
|
89
90
|
History.txt
|
90
|
-
Manifest.txt
|
91
91
|
java/antlr-full-3.2.1.jar
|
92
92
|
java/RubyTarget.java
|
93
93
|
Manifest.txt
|
data/java/RubyTarget.java
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
/*
|
2
2
|
[The "BSD licence"]
|
3
|
-
Copyright (c)
|
3
|
+
Copyright (c) 2010 Martin Traverso
|
4
4
|
All rights reserved.
|
5
5
|
|
6
6
|
Redistribution and use in source and binary forms, with or without
|
@@ -36,377 +36,387 @@ import org.antlr.stringtemplate.*;
|
|
36
36
|
import org.antlr.tool.Grammar;
|
37
37
|
|
38
38
|
public class RubyTarget
|
39
|
-
extends Target
|
39
|
+
extends Target
|
40
40
|
{
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
public String toString(Object o, String formatName) {
|
68
|
-
String idString = o.toString();
|
69
|
-
|
70
|
-
if (idString.isEmpty()) return idString;
|
71
|
-
|
72
|
-
if (formatName.equals("snakecase")) {
|
73
|
-
return snakecase(idString);
|
74
|
-
} else if (formatName.equals("camelcase")) {
|
75
|
-
return camelcase(idString);
|
76
|
-
} else if (formatName.equals("subcamelcase")) {
|
77
|
-
return subcamelcase(idString);
|
78
|
-
} else if (formatName.equals("constant")) {
|
79
|
-
return constantcase(idString);
|
80
|
-
} else if (formatName.equals("platform")) {
|
81
|
-
return platform(idString);
|
82
|
-
} else if (formatName.equals("lexerRule")) {
|
83
|
-
return lexerRule(idString);
|
84
|
-
} else if (formatName.equals("constantPath")) {
|
85
|
-
return constantPath(idString);
|
86
|
-
} else if (formatName.equals("label")) {
|
87
|
-
return label(idString);
|
88
|
-
} else if (formatName.equals("symbol")) {
|
89
|
-
return symbol(idString);
|
90
|
-
} else {
|
91
|
-
throw new IllegalArgumentException("Unsupported format name");
|
92
|
-
}
|
93
|
-
}
|
94
|
-
/** given an input string, which is presumed
|
95
|
-
* to contain a word, which may potentially be camelcased,
|
96
|
-
* and convert it to snake_case underscore style.
|
97
|
-
*
|
98
|
-
* algorithm --
|
99
|
-
* iterate through the string with a sliding window 3 chars wide
|
100
|
-
*
|
101
|
-
* example -- aGUIWhatNot
|
102
|
-
* c c+1 c+2 action
|
103
|
-
* a G << 'a' << '_' // a lower-upper word edge
|
104
|
-
* G U I << 'g'
|
105
|
-
* U I W << 'w'
|
106
|
-
* I W h << 'i' << '_' // the last character in an acronym run of uppers
|
107
|
-
* W h << 'w'
|
108
|
-
* ... and so on
|
109
|
-
*/
|
110
|
-
private String snakecase(String value) {
|
111
|
-
StringBuilder output_buffer = new StringBuilder();
|
112
|
-
int l = value.length();
|
113
|
-
int cliff = l - 1;
|
114
|
-
char cur;
|
115
|
-
char next;
|
116
|
-
char peek;
|
117
|
-
|
118
|
-
if (value.isEmpty()) return value;
|
119
|
-
if (l == 1) return value.toLowerCase();
|
120
|
-
|
121
|
-
for (int i = 0; i < cliff; i++) {
|
122
|
-
cur = value.charAt(i);
|
123
|
-
next = value.charAt(i + 1);
|
124
|
-
|
125
|
-
if ( Character.isLetter( cur ) ) {
|
126
|
-
output_buffer.append( Character.toLowerCase( cur ) );
|
127
|
-
|
128
|
-
if ( Character.isDigit( next ) || Character.isWhitespace( next ) ) {
|
129
|
-
output_buffer.append( '_' );
|
130
|
-
} else if ( Character.isLowerCase( cur ) && Character.isUpperCase( next ) ) {
|
131
|
-
// at camelcase word edge
|
132
|
-
output_buffer.append( '_' );
|
133
|
-
} else if ( (i < cliff - 1) && Character.isUpperCase( cur ) && Character.isUpperCase( next ) ) {
|
134
|
-
// cur is part of an acronym
|
135
|
-
|
136
|
-
peek = value.charAt(i + 2);
|
137
|
-
if ( Character.isLowerCase( peek ) ) {
|
138
|
-
/* if next is the start of word (indicated when peek is lowercase)
|
139
|
-
then the acronym must be completed by appending an underscore */
|
140
|
-
output_buffer.append('_');
|
141
|
-
}
|
142
|
-
}
|
143
|
-
} else if( Character.isDigit( cur ) ) {
|
144
|
-
output_buffer.append( cur );
|
145
|
-
if ( Character.isLetter( next ) ) {
|
146
|
-
output_buffer.append('_');
|
147
|
-
}
|
148
|
-
} else if (Character.isWhitespace( cur )) {
|
149
|
-
// do nothing
|
150
|
-
} else {
|
151
|
-
output_buffer.append( cur );
|
152
|
-
}
|
153
|
-
|
154
|
-
}
|
155
|
-
|
156
|
-
cur = value.charAt(cliff);
|
157
|
-
if (! Character.isWhitespace(cur) ) {
|
158
|
-
output_buffer.append( Character.toLowerCase( cur ) );
|
159
|
-
}
|
160
|
-
|
161
|
-
return output_buffer.toString();
|
162
|
-
}
|
163
|
-
private String constantcase(String value) {
|
164
|
-
return snakecase(value).toUpperCase();
|
165
|
-
}
|
166
|
-
private String platform(String value) {
|
167
|
-
return ("__" + value + "__");
|
168
|
-
}
|
169
|
-
private String symbol(String value) {
|
170
|
-
if (value.matches("[a-zA-Z_]\\w*[\\?\\!\\=]?")) {
|
171
|
-
return (":" + value);
|
172
|
-
} else {
|
173
|
-
return ("%s(" + value + ")");
|
174
|
-
}
|
175
|
-
}
|
176
|
-
private String lexerRule(String value) {
|
177
|
-
if (value.equals("Tokens")) {
|
178
|
-
return "token!";
|
179
|
-
} else {
|
180
|
-
return (snakecase(value) + "!");
|
181
|
-
}
|
182
|
-
}
|
183
|
-
private String constantPath(String value) {
|
184
|
-
return value.replaceAll("\\.", "::");
|
185
|
-
}
|
186
|
-
private String camelcase(String value) {
|
187
|
-
StringBuilder output_buffer = new StringBuilder();
|
188
|
-
int cliff = value.length();
|
189
|
-
char cur;
|
190
|
-
char next;
|
191
|
-
boolean at_edge = true;
|
192
|
-
|
193
|
-
if (value.isEmpty()) return value;
|
194
|
-
if (cliff == 1) return value.toUpperCase();
|
195
|
-
|
196
|
-
for (int i = 0; i < cliff; i++) {
|
197
|
-
cur = value.charAt(i);
|
198
|
-
|
199
|
-
if ( Character.isWhitespace( cur ) ) {
|
200
|
-
at_edge = true;
|
201
|
-
continue;
|
202
|
-
} else if ( cur == '_' ) {
|
203
|
-
at_edge = true;
|
204
|
-
continue;
|
205
|
-
} else if ( Character.isDigit( cur ) ) {
|
206
|
-
output_buffer.append( cur );
|
207
|
-
at_edge = true;
|
208
|
-
continue;
|
209
|
-
}
|
210
|
-
|
211
|
-
if (at_edge) {
|
212
|
-
output_buffer.append( Character.toUpperCase( cur ) );
|
213
|
-
if ( Character.isLetter( cur ) ) at_edge = false;
|
214
|
-
} else {
|
215
|
-
output_buffer.append( cur );
|
216
|
-
}
|
217
|
-
}
|
218
|
-
|
219
|
-
return output_buffer.toString();
|
220
|
-
}
|
221
|
-
private String label(String value) {
|
222
|
-
if (rubyKeywords.contains(value)) {
|
223
|
-
return platform(value);
|
224
|
-
} else if (Character.isUpperCase(value.charAt(0)) &&
|
225
|
-
(!value.equals("FILE")) &&
|
226
|
-
(!value.equals("LINE"))) {
|
227
|
-
return platform(value);
|
228
|
-
} else if (value.equals("FILE")) {
|
229
|
-
return "_FILE_";
|
230
|
-
} else if (value.equals("LINE")) {
|
231
|
-
return "_LINE_";
|
232
|
-
} else {
|
233
|
-
return value;
|
234
|
-
}
|
235
|
-
}
|
236
|
-
private String subcamelcase(String value) {
|
237
|
-
value = camelcase(value);
|
238
|
-
if (value.isEmpty())
|
239
|
-
return value;
|
240
|
-
Character head = Character.toLowerCase( value.charAt(0) );
|
241
|
-
String tail = value.substring(1);
|
242
|
-
return head.toString().concat(tail);
|
243
|
-
}
|
244
|
-
}
|
245
|
-
|
246
|
-
protected void genRecognizerFile(Tool tool,
|
247
|
-
CodeGenerator generator,
|
248
|
-
Grammar grammar,
|
249
|
-
StringTemplate outputFileST)
|
250
|
-
throws IOException
|
251
|
-
{
|
252
|
-
StringTemplateGroup group = generator.getTemplates();
|
253
|
-
RubyRenderer renderer = new RubyRenderer();
|
254
|
-
try {
|
255
|
-
group.registerRenderer(Class.forName("java.lang.String"), renderer);
|
256
|
-
} catch (ClassNotFoundException e) {
|
257
|
-
// this shouldn't happen
|
258
|
-
System.err.println("ClassNotFoundException: " + e.getMessage());
|
259
|
-
e.printStackTrace(System.err);
|
260
|
-
}
|
261
|
-
String fileName =
|
262
|
-
generator.getRecognizerFileName(grammar.name, grammar.type);
|
263
|
-
generator.write(outputFileST, fileName);
|
264
|
-
}
|
265
|
-
|
266
|
-
public String getTargetCharLiteralFromANTLRCharLiteral(
|
267
|
-
CodeGenerator generator,
|
268
|
-
String literal)
|
269
|
-
{
|
270
|
-
literal = literal.substring(1, literal.length() - 1);
|
271
|
-
|
272
|
-
String result = "?";
|
273
|
-
|
274
|
-
if (literal.equals("\\")) {
|
275
|
-
result += "\\\\";
|
276
|
-
}
|
277
|
-
else if (literal.equals(" ")) {
|
278
|
-
result += "\\s";
|
279
|
-
}
|
280
|
-
else if (literal.startsWith("\\u")) {
|
281
|
-
result = "0x" + literal.substring(2);
|
282
|
-
}
|
283
|
-
else {
|
284
|
-
result += literal;
|
285
|
-
}
|
286
|
-
|
287
|
-
return result;
|
288
|
-
}
|
289
|
-
|
290
|
-
public int getMaxCharValue(CodeGenerator generator)
|
291
|
-
{
|
292
|
-
// Versions before 1.9 do not support unicode
|
293
|
-
return 0xFF;
|
294
|
-
}
|
295
|
-
|
296
|
-
public String getTokenTypeAsTargetLabel(CodeGenerator generator, int ttype)
|
297
|
-
{
|
298
|
-
String name = generator.grammar.getTokenDisplayName(ttype);
|
299
|
-
// If name is a literal, return the token type instead
|
300
|
-
if ( name.charAt(0)=='\'' ) {
|
301
|
-
return generator.grammar.computeTokenNameFromLiteral(ttype, name);
|
302
|
-
}
|
303
|
-
return name;
|
304
|
-
}
|
305
|
-
|
306
|
-
/** Is scope in @scope::name {action} valid for this kind of grammar?
|
307
|
-
* Targets like C++ may want to allow new scopes like headerfile or
|
308
|
-
* some such. The action names themselves are not policed at the
|
309
|
-
* moment so targets can add template actions w/o having to recompile
|
310
|
-
* ANTLR.
|
311
|
-
*/
|
312
|
-
public boolean isValidActionScope(int grammarType, String scope) {
|
313
|
-
switch (grammarType) {
|
314
|
-
case Grammar.LEXER:
|
315
|
-
if (scope.equals("lexer")) {
|
316
|
-
return true;
|
317
|
-
}
|
318
|
-
if (scope.equals("token")) {
|
319
|
-
return true;
|
320
|
-
}
|
321
|
-
if (scope.equals("module")) {
|
322
|
-
return true;
|
323
|
-
}
|
324
|
-
if (scope.equals("overrides")) {
|
325
|
-
return true;
|
326
|
-
}
|
327
|
-
break;
|
328
|
-
case Grammar.PARSER:
|
329
|
-
if (scope.equals("parser")) {
|
330
|
-
return true;
|
331
|
-
}
|
332
|
-
if (scope.equals("token")) {
|
333
|
-
return true;
|
334
|
-
}
|
335
|
-
if (scope.equals("module")) {
|
336
|
-
return true;
|
337
|
-
}
|
338
|
-
if (scope.equals("overrides")) {
|
339
|
-
return true;
|
340
|
-
}
|
341
|
-
break;
|
342
|
-
case Grammar.COMBINED:
|
343
|
-
if (scope.equals("parser")) {
|
344
|
-
return true;
|
345
|
-
}
|
346
|
-
if (scope.equals("lexer")) {
|
347
|
-
return true;
|
348
|
-
}
|
349
|
-
if (scope.equals("token")) {
|
350
|
-
return true;
|
351
|
-
}
|
352
|
-
if (scope.equals("module")) {
|
353
|
-
return true;
|
354
|
-
}
|
355
|
-
if (scope.equals("overrides")) {
|
356
|
-
return true;
|
357
|
-
}
|
358
|
-
break;
|
359
|
-
case Grammar.TREE_PARSER:
|
360
|
-
if (scope.equals("treeparser")) {
|
361
|
-
return true;
|
362
|
-
}
|
363
|
-
if (scope.equals("token")) {
|
364
|
-
return true;
|
365
|
-
}
|
366
|
-
if (scope.equals("module")) {
|
367
|
-
return true;
|
368
|
-
}
|
369
|
-
if (scope.equals("overrides")) {
|
370
|
-
return true;
|
371
|
-
}
|
372
|
-
break;
|
373
|
-
}
|
374
|
-
return false;
|
375
|
-
}
|
376
|
-
|
377
|
-
/*
|
378
|
-
public String getTargetStringLiteralFromString(String s)
|
41
|
+
/** A set of ruby keywords which are used to escape labels and method names
|
42
|
+
* which will cause parse errors in the ruby source
|
43
|
+
*/
|
44
|
+
public static final Set rubyKeywords =
|
45
|
+
new HashSet() {
|
46
|
+
{
|
47
|
+
add( "alias" ); add( "END" ); add( "retry" );
|
48
|
+
add( "and" ); add( "ensure" ); add( "return" );
|
49
|
+
add( "BEGIN" ); add( "false" ); add( "self" );
|
50
|
+
add( "begin" ); add( "for" ); add( "super" );
|
51
|
+
add( "break" ); add( "if" ); add( "then" );
|
52
|
+
add( "case" ); add( "in" ); add( "true" );
|
53
|
+
add( "class" ); add( "module" ); add( "undef" );
|
54
|
+
add( "def" ); add( "next" ); add( "unless" );
|
55
|
+
add( "defined?" ); add( "nil" ); add( "until" );
|
56
|
+
add( "do" ); add( "not" ); add( "when" );
|
57
|
+
add( "else" ); add( "or" ); add( "while" );
|
58
|
+
add( "elsif" ); add( "redo" ); add( "yield" );
|
59
|
+
add( "end" ); add( "rescue" );
|
60
|
+
}
|
61
|
+
};
|
62
|
+
|
63
|
+
public static HashMap sharedActionBlocks = new HashMap();
|
64
|
+
|
65
|
+
public class RubyRenderer
|
66
|
+
implements AttributeRenderer
|
379
67
|
{
|
380
|
-
|
381
|
-
|
68
|
+
public String toString( Object o ) {
|
69
|
+
return o.toString();
|
70
|
+
}
|
71
|
+
|
72
|
+
public String toString( Object o, String formatName ) {
|
73
|
+
String idString = o.toString();
|
74
|
+
|
75
|
+
if ( idString.isEmpty() ) return idString;
|
76
|
+
|
77
|
+
if ( formatName.equals( "snakecase" ) ) {
|
78
|
+
return snakecase( idString );
|
79
|
+
} else if ( formatName.equals( "camelcase" ) ) {
|
80
|
+
return camelcase( idString );
|
81
|
+
} else if ( formatName.equals( "subcamelcase" ) ) {
|
82
|
+
return subcamelcase( idString );
|
83
|
+
} else if ( formatName.equals( "constant" ) ) {
|
84
|
+
return constantcase( idString );
|
85
|
+
} else if ( formatName.equals( "platform" ) ) {
|
86
|
+
return platform( idString );
|
87
|
+
} else if ( formatName.equals( "lexerRule" ) ) {
|
88
|
+
return lexerRule( idString );
|
89
|
+
} else if ( formatName.equals( "constantPath" ) ) {
|
90
|
+
return constantPath( idString );
|
91
|
+
} else if ( formatName.equals( "label" ) ) {
|
92
|
+
return label( idString );
|
93
|
+
} else if ( formatName.equals( "symbol" ) ) {
|
94
|
+
return symbol( idString );
|
95
|
+
} else {
|
96
|
+
throw new IllegalArgumentException( "Unsupported format name" );
|
97
|
+
}
|
98
|
+
}
|
99
|
+
|
100
|
+
/** given an input string, which is presumed
|
101
|
+
* to contain a word, which may potentially be camelcased,
|
102
|
+
* and convert it to snake_case underscore style.
|
103
|
+
*
|
104
|
+
* algorithm --
|
105
|
+
* iterate through the string with a sliding window 3 chars wide
|
106
|
+
*
|
107
|
+
* example -- aGUIWhatNot
|
108
|
+
* c c+1 c+2 action
|
109
|
+
* a G << 'a' << '_' // a lower-upper word edge
|
110
|
+
* G U I << 'g'
|
111
|
+
* U I W << 'w'
|
112
|
+
* I W h << 'i' << '_' // the last character in an acronym run of uppers
|
113
|
+
* W h << 'w'
|
114
|
+
* ... and so on
|
115
|
+
*/
|
116
|
+
private String snakecase( String value ) {
|
117
|
+
StringBuilder output_buffer = new StringBuilder();
|
118
|
+
int l = value.length();
|
119
|
+
int cliff = l - 1;
|
120
|
+
char cur;
|
121
|
+
char next;
|
122
|
+
char peek;
|
123
|
+
|
124
|
+
if ( value.isEmpty() ) return value;
|
125
|
+
if ( l == 1 ) return value.toLowerCase();
|
126
|
+
|
127
|
+
for ( int i = 0; i < cliff; i++ ) {
|
128
|
+
cur = value.charAt( i );
|
129
|
+
next = value.charAt( i + 1 );
|
130
|
+
|
131
|
+
if ( Character.isLetter( cur ) ) {
|
132
|
+
output_buffer.append( Character.toLowerCase( cur ) );
|
133
|
+
|
134
|
+
if ( Character.isDigit( next ) || Character.isWhitespace( next ) ) {
|
135
|
+
output_buffer.append( '_' );
|
136
|
+
} else if ( Character.isLowerCase( cur ) && Character.isUpperCase( next ) ) {
|
137
|
+
// at camelcase word edge
|
138
|
+
output_buffer.append( '_' );
|
139
|
+
} else if ( ( i < cliff - 1 ) && Character.isUpperCase( cur ) && Character.isUpperCase( next ) ) {
|
140
|
+
// cur is part of an acronym
|
141
|
+
|
142
|
+
peek = value.charAt( i + 2 );
|
143
|
+
if ( Character.isLowerCase( peek ) ) {
|
144
|
+
/* if next is the start of word (indicated when peek is lowercase)
|
145
|
+
then the acronym must be completed by appending an underscore */
|
146
|
+
output_buffer.append( '_' );
|
147
|
+
}
|
148
|
+
}
|
149
|
+
} else if ( Character.isDigit( cur ) ) {
|
150
|
+
output_buffer.append( cur );
|
151
|
+
if ( Character.isLetter( next ) ) {
|
152
|
+
output_buffer.append( '_' );
|
153
|
+
}
|
154
|
+
} else if ( Character.isWhitespace( cur ) ) {
|
155
|
+
// do nothing
|
156
|
+
} else {
|
157
|
+
output_buffer.append( cur );
|
158
|
+
}
|
159
|
+
|
160
|
+
}
|
161
|
+
|
162
|
+
cur = value.charAt( cliff );
|
163
|
+
if ( ! Character.isWhitespace( cur ) ) {
|
164
|
+
output_buffer.append( Character.toLowerCase( cur ) );
|
165
|
+
}
|
166
|
+
|
167
|
+
return output_buffer.toString();
|
168
|
+
}
|
169
|
+
|
170
|
+
private String constantcase( String value ) {
|
171
|
+
return snakecase( value ).toUpperCase();
|
172
|
+
}
|
173
|
+
|
174
|
+
private String platform( String value ) {
|
175
|
+
return ( "__" + value + "__" );
|
176
|
+
}
|
177
|
+
|
178
|
+
private String symbol( String value ) {
|
179
|
+
if ( value.matches( "[a-zA-Z_]\\w*[\\?\\!\\=]?" ) ) {
|
180
|
+
return ( ":" + value );
|
181
|
+
} else {
|
182
|
+
return ( "%s(" + value + ")" );
|
183
|
+
}
|
184
|
+
}
|
185
|
+
|
186
|
+
private String lexerRule( String value ) {
|
187
|
+
// System.out.print( "lexerRule( \"" + value + "\") => " );
|
188
|
+
if ( value.equals( "Tokens" ) ) {
|
189
|
+
// System.out.println( "\"token!\"" );
|
190
|
+
return "token!";
|
191
|
+
} else {
|
192
|
+
// String result = snakecase( value ) + "!";
|
193
|
+
// System.out.println( "\"" + result + "\"" );
|
194
|
+
return ( snakecase( value ) + "!" );
|
195
|
+
}
|
196
|
+
}
|
197
|
+
|
198
|
+
private String constantPath( String value ) {
|
199
|
+
return value.replaceAll( "\\.", "::" );
|
200
|
+
}
|
201
|
+
|
202
|
+
private String camelcase( String value ) {
|
203
|
+
StringBuilder output_buffer = new StringBuilder();
|
204
|
+
int cliff = value.length();
|
205
|
+
char cur;
|
206
|
+
char next;
|
207
|
+
boolean at_edge = true;
|
208
|
+
|
209
|
+
if ( value.isEmpty() ) return value;
|
210
|
+
if ( cliff == 1 ) return value.toUpperCase();
|
211
|
+
|
212
|
+
for ( int i = 0; i < cliff; i++ ) {
|
213
|
+
cur = value.charAt( i );
|
214
|
+
|
215
|
+
if ( Character.isWhitespace( cur ) ) {
|
216
|
+
at_edge = true;
|
217
|
+
continue;
|
218
|
+
} else if ( cur == '_' ) {
|
219
|
+
at_edge = true;
|
220
|
+
continue;
|
221
|
+
} else if ( Character.isDigit( cur ) ) {
|
222
|
+
output_buffer.append( cur );
|
223
|
+
at_edge = true;
|
224
|
+
continue;
|
225
|
+
}
|
226
|
+
|
227
|
+
if ( at_edge ) {
|
228
|
+
output_buffer.append( Character.toUpperCase( cur ) );
|
229
|
+
if ( Character.isLetter( cur ) ) at_edge = false;
|
230
|
+
} else {
|
231
|
+
output_buffer.append( cur );
|
232
|
+
}
|
233
|
+
}
|
234
|
+
|
235
|
+
return output_buffer.toString();
|
236
|
+
}
|
237
|
+
|
238
|
+
private String label( String value ) {
|
239
|
+
if ( rubyKeywords.contains( value ) ) {
|
240
|
+
return platform( value );
|
241
|
+
} else if ( Character.isUpperCase( value.charAt( 0 ) ) &&
|
242
|
+
( !value.equals( "FILE" ) ) &&
|
243
|
+
( !value.equals( "LINE" ) ) ) {
|
244
|
+
return platform( value );
|
245
|
+
} else if ( value.equals( "FILE" ) ) {
|
246
|
+
return "_FILE_";
|
247
|
+
} else if ( value.equals( "LINE" ) ) {
|
248
|
+
return "_LINE_";
|
249
|
+
} else {
|
250
|
+
return value;
|
251
|
+
}
|
252
|
+
}
|
253
|
+
|
254
|
+
private String subcamelcase( String value ) {
|
255
|
+
value = camelcase( value );
|
256
|
+
if ( value.isEmpty() )
|
257
|
+
return value;
|
258
|
+
Character head = Character.toLowerCase( value.charAt( 0 ) );
|
259
|
+
String tail = value.substring( 1 );
|
260
|
+
return head.toString().concat( tail );
|
261
|
+
}
|
382
262
|
}
|
383
263
|
|
384
|
-
|
264
|
+
protected void genRecognizerFile(
|
265
|
+
Tool tool,
|
266
|
+
CodeGenerator generator,
|
267
|
+
Grammar grammar,
|
268
|
+
StringTemplate outputFileST
|
269
|
+
) throws IOException
|
385
270
|
{
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
271
|
+
/*
|
272
|
+
Below is an experimental attempt at providing a few named action blocks
|
273
|
+
that are printed in both lexer and parser files from combined grammars.
|
274
|
+
ANTLR appears to first generate a parser, then generate an independent lexer,
|
275
|
+
and then generate code from that. It keeps the combo/parser grammar object
|
276
|
+
and the lexer grammar object, as well as their respective code generator and
|
277
|
+
target instances, completely independent. So, while a bit hack-ish, this is
|
278
|
+
a solution that should work without having to modify Terrence Parr's
|
279
|
+
core tool code.
|
280
|
+
|
281
|
+
- sharedActionBlocks is a class variable containing a hash map
|
282
|
+
- if this method is called with a combo grammar, and the action map
|
283
|
+
in the grammar contains an entry for the named scope "all",
|
284
|
+
add an entry to sharedActionBlocks mapping the grammar name to
|
285
|
+
the "all" action map.
|
286
|
+
- if this method is called with an `implicit lexer'
|
287
|
+
(one that's extracted from a combo grammar), check to see if
|
288
|
+
there's an entry in sharedActionBlocks for the lexer's grammar name.
|
289
|
+
- if there is an action map entry, place it in the lexer's action map
|
290
|
+
- the recognizerFile template has code to place the
|
291
|
+
"all" actions appropriately
|
292
|
+
|
293
|
+
problems:
|
294
|
+
- This solution assumes that the parser will be generated
|
295
|
+
before the lexer. If that changes at some point, this will
|
296
|
+
not work.
|
297
|
+
- I have not investigated how this works with delegation yet
|
298
|
+
|
299
|
+
Kyle Yetter - March 25, 2010
|
300
|
+
*/
|
301
|
+
|
302
|
+
if ( grammar.type == Grammar.COMBINED ) {
|
303
|
+
Map actions = grammar.getActions();
|
304
|
+
if ( actions.containsKey( "all" ) ) {
|
305
|
+
sharedActionBlocks.put( grammar.name, actions.get( "all" ) );
|
306
|
+
}
|
307
|
+
} else if ( grammar.implicitLexer ) {
|
308
|
+
if ( sharedActionBlocks.containsKey( grammar.name ) ) {
|
309
|
+
Map actions = grammar.getActions();
|
310
|
+
actions.put( "all", sharedActionBlocks.get( grammar.name ) );
|
311
|
+
}
|
312
|
+
}
|
313
|
+
|
314
|
+
StringTemplateGroup group = generator.getTemplates();
|
315
|
+
RubyRenderer renderer = new RubyRenderer();
|
316
|
+
try {
|
317
|
+
group.registerRenderer( Class.forName( "java.lang.String" ), renderer );
|
318
|
+
} catch ( ClassNotFoundException e ) {
|
319
|
+
// this shouldn't happen
|
320
|
+
System.err.println( "ClassNotFoundException: " + e.getMessage() );
|
321
|
+
e.printStackTrace( System.err );
|
322
|
+
}
|
323
|
+
String fileName =
|
324
|
+
generator.getRecognizerFileName( grammar.name, grammar.type );
|
325
|
+
generator.write( outputFileST, fileName );
|
390
326
|
}
|
391
327
|
|
392
|
-
public String
|
328
|
+
public String getTargetCharLiteralFromANTLRCharLiteral(
|
329
|
+
CodeGenerator generator,
|
330
|
+
String literal
|
331
|
+
)
|
393
332
|
{
|
394
|
-
|
395
|
-
|
396
|
-
|
333
|
+
literal = literal.substring( 1, literal.length() - 1 );
|
334
|
+
|
335
|
+
String result = "?";
|
336
|
+
|
337
|
+
if ( literal.equals( "\\" ) ) {
|
338
|
+
result += "\\\\";
|
339
|
+
}
|
340
|
+
else if ( literal.equals( " " ) ) {
|
341
|
+
result += "\\s";
|
342
|
+
}
|
343
|
+
else if ( literal.startsWith( "\\u" ) ) {
|
344
|
+
result = "0x" + literal.substring( 2 );
|
345
|
+
}
|
346
|
+
else {
|
347
|
+
result += literal;
|
348
|
+
}
|
349
|
+
|
397
350
|
return result;
|
398
351
|
}
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
352
|
+
|
353
|
+
public int getMaxCharValue( CodeGenerator generator )
|
354
|
+
{
|
355
|
+
// Versions before 1.9 do not support unicode
|
356
|
+
return 0xFF;
|
357
|
+
}
|
358
|
+
|
359
|
+
public String getTokenTypeAsTargetLabel( CodeGenerator generator, int ttype )
|
360
|
+
{
|
361
|
+
String name = generator.grammar.getTokenDisplayName( ttype );
|
362
|
+
// If name is a literal, return the token type instead
|
363
|
+
if ( name.charAt( 0 )=='\'' ) {
|
364
|
+
return generator.grammar.computeTokenNameFromLiteral( ttype, name );
|
365
|
+
}
|
366
|
+
return name;
|
367
|
+
}
|
368
|
+
|
369
|
+
public boolean isValidActionScope( int grammarType, String scope ) {
|
370
|
+
if ( scope.equals( "all" ) ) {
|
371
|
+
return true;
|
372
|
+
}
|
373
|
+
if ( scope.equals( "token" ) ) {
|
374
|
+
return true;
|
375
|
+
}
|
376
|
+
if ( scope.equals( "module" ) ) {
|
377
|
+
return true;
|
378
|
+
}
|
379
|
+
if ( scope.equals( "overrides" ) ) {
|
380
|
+
return true;
|
381
|
+
}
|
382
|
+
|
383
|
+
switch ( grammarType ) {
|
384
|
+
case Grammar.LEXER:
|
385
|
+
if ( scope.equals( "lexer" ) ) {
|
386
|
+
return true;
|
387
|
+
}
|
388
|
+
break;
|
389
|
+
case Grammar.PARSER:
|
390
|
+
if ( scope.equals( "parser" ) ) {
|
391
|
+
return true;
|
392
|
+
}
|
393
|
+
break;
|
394
|
+
case Grammar.COMBINED:
|
395
|
+
if ( scope.equals( "parser" ) ) {
|
396
|
+
return true;
|
397
|
+
}
|
398
|
+
if ( scope.equals( "lexer" ) ) {
|
399
|
+
return true;
|
400
|
+
}
|
401
|
+
break;
|
402
|
+
case Grammar.TREE_PARSER:
|
403
|
+
if ( scope.equals( "treeparser" ) ) {
|
404
|
+
return true;
|
405
|
+
}
|
406
|
+
break;
|
407
|
+
}
|
408
|
+
return false;
|
409
|
+
}
|
410
|
+
|
411
|
+
public String encodeIntAsCharEscape( final int v ) {
|
412
|
+
final int intValue;
|
413
|
+
|
414
|
+
if ( v == 65535 ) {
|
415
|
+
intValue = -1;
|
416
|
+
} else {
|
417
|
+
intValue = v;
|
418
|
+
}
|
419
|
+
|
420
|
+
return String.valueOf( intValue );
|
421
|
+
}
|
412
422
|
}
|