antlr3 1.2.4 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/antlr4ruby +1 -13
- data/java/RubyTarget.java +8 -125
- data/java/antlr-full-3.2.1.jar +0 -0
- data/lib/antlr3/constants.rb +2 -0
- data/lib/antlr3/dfa.rb +18 -15
- data/lib/antlr3/error.rb +3 -3
- data/lib/antlr3/main.rb +8 -9
- data/lib/antlr3/modes/ast-builder.rb +24 -4
- data/lib/antlr3/recognizers.rb +17 -16
- data/lib/antlr3/task.rb +13 -0
- data/lib/antlr3/token.rb +53 -17
- data/lib/antlr3/tree.rb +491 -561
- data/lib/antlr3/tree/wizard.rb +139 -127
- data/lib/antlr3/util.rb +7 -1
- data/lib/antlr3/version.rb +4 -4
- data/samples/{Cpp.g → CPP.g} +1 -1
- data/templates/AST.stg +1 -1
- data/templates/Ruby.stg +29 -42
- data/test/functional/ast-output/auto-ast.rb +3 -3
- data/test/functional/ast-output/tree-rewrite.rb +1 -1
- data/test/functional/debugging/debug-mode.rb +2 -2
- data/test/unit/test-tree-wizard.rb +115 -156
- data/test/unit/test-trees.rb +66 -77
- metadata +4 -5
- data/lib/antlr3/test/config.rb +0 -23
- data/lib/antlr3/test/diff.rb +0 -165
data/lib/antlr3/tree/wizard.rb
CHANGED
@@ -82,8 +82,8 @@ for more background on the concept of a tree wizard.
|
|
82
82
|
# => nil - invalid node names
|
83
83
|
|
84
84
|
# test whether a tree matches a pattern
|
85
|
-
wizard.
|
86
|
-
wizard.
|
85
|
+
wizard.match(expression_node, '(MINUS VAR .)') # => true
|
86
|
+
wizard.match(lone_node, 'NUMBER NUMBER') # => false
|
87
87
|
|
88
88
|
# extract nodes matching a pattern
|
89
89
|
wizard.find(statement_node, '(PLUS . .)')
|
@@ -116,15 +116,16 @@ for more background on the concept of a tree wizard.
|
|
116
116
|
|
117
117
|
class Wizard
|
118
118
|
|
119
|
-
include
|
119
|
+
include Constants
|
120
|
+
include Util
|
120
121
|
|
121
|
-
=begin rdoc ANTLR3::AST::Wizard::
|
122
|
+
=begin rdoc ANTLR3::AST::Wizard::PatternLexer
|
122
123
|
|
123
124
|
A class that is used internally by AST::Wizard to tokenize tree patterns
|
124
125
|
|
125
126
|
=end
|
126
127
|
|
127
|
-
class
|
128
|
+
class PatternLexer
|
128
129
|
include ANTLR3::Constants
|
129
130
|
|
130
131
|
autoload :StringScanner, 'strscan'
|
@@ -141,9 +142,9 @@ A class that is used internally by AST::Wizard to tokenize tree patterns
|
|
141
142
|
]
|
142
143
|
|
143
144
|
attr_reader :text, :error, :pattern
|
144
|
-
def initialize(pattern)
|
145
|
+
def initialize( pattern )
|
145
146
|
@pattern = pattern.to_s
|
146
|
-
@scanner = StringScanner.new(pattern)
|
147
|
+
@scanner = StringScanner.new( pattern )
|
147
148
|
@text = ''
|
148
149
|
@error = false
|
149
150
|
end
|
@@ -174,19 +175,24 @@ A class that is used internally by AST::Wizard to tokenize tree patterns
|
|
174
175
|
end
|
175
176
|
|
176
177
|
|
177
|
-
=begin rdoc ANTLR3::AST::Wizard::
|
178
|
+
=begin rdoc ANTLR3::AST::Wizard::Pattern
|
178
179
|
|
179
180
|
A class that is used internally by AST::Wizard to construct AST tree objects
|
180
181
|
from a tokenized tree pattern
|
181
182
|
|
182
183
|
=end
|
183
184
|
|
184
|
-
class
|
185
|
+
class PatternParser
|
186
|
+
def self.parse( pattern, token_scheme, adaptor )
|
187
|
+
lexer = PatternLexer.new( pattern )
|
188
|
+
new( lexer, token_scheme, adaptor ).pattern
|
189
|
+
end
|
190
|
+
|
185
191
|
include ANTLR3::Constants
|
186
192
|
|
187
|
-
def initialize(tokenizer,
|
193
|
+
def initialize( tokenizer, token_scheme, adaptor )
|
188
194
|
@tokenizer = tokenizer
|
189
|
-
@
|
195
|
+
@token_scheme = token_scheme
|
190
196
|
@adaptor = adaptor
|
191
197
|
@token_type = tokenizer.next_token
|
192
198
|
end
|
@@ -212,7 +218,7 @@ from a tokenized tree pattern
|
|
212
218
|
loop do
|
213
219
|
case @token_type
|
214
220
|
when :open
|
215
|
-
subtree = parse_tree
|
221
|
+
subtree = parse_tree
|
216
222
|
@adaptor.add_child(root, subtree)
|
217
223
|
when :identifier, :percent, :dot
|
218
224
|
child = parse_node or return nil
|
@@ -237,7 +243,7 @@ from a tokenized tree pattern
|
|
237
243
|
if @token_type == :dot
|
238
244
|
@token_type = @tokenizer.next_token
|
239
245
|
wildcard_payload = CommonToken.create(:type => 0, :text => '.')
|
240
|
-
node =
|
246
|
+
node = WildcardPattern.new(wildcard_payload)
|
241
247
|
label and node.label = label
|
242
248
|
return node
|
243
249
|
end
|
@@ -255,190 +261,188 @@ from a tokenized tree pattern
|
|
255
261
|
@token_type = @tokenizer.next_token
|
256
262
|
end
|
257
263
|
|
258
|
-
|
259
|
-
|
264
|
+
node_type = @token_scheme[ token_name ] || INVALID_TOKEN_TYPE
|
265
|
+
node = @adaptor.create_from_type!( node_type, text )
|
260
266
|
|
261
|
-
|
262
|
-
|
263
|
-
|
267
|
+
if Pattern === node
|
268
|
+
node.label, node.has_text_arg = label, arg
|
269
|
+
end
|
264
270
|
return node
|
265
271
|
end
|
266
272
|
end
|
267
273
|
|
268
274
|
|
269
|
-
=begin rdoc ANTLR3::AST::Wizard::
|
275
|
+
=begin rdoc ANTLR3::AST::Wizard::Pattern
|
270
276
|
|
271
277
|
A simple tree class that represents the skeletal structure of tree. It is used
|
272
278
|
to validate tree structures as well as to extract nodes that match the pattern.
|
273
279
|
|
274
280
|
=end
|
275
281
|
|
276
|
-
class
|
282
|
+
class Pattern < CommonTree
|
283
|
+
def self.parse( pattern_str, scheme )
|
284
|
+
PatternParser.parse(
|
285
|
+
pattern_str, scheme, PatternAdaptor.new( scheme.token_class )
|
286
|
+
)
|
287
|
+
end
|
288
|
+
|
277
289
|
attr_accessor :label, :has_text_arg
|
278
|
-
|
279
|
-
|
290
|
+
alias :has_text_arg? :has_text_arg
|
291
|
+
|
292
|
+
def initialize( payload )
|
293
|
+
super( payload )
|
280
294
|
@label = nil
|
281
295
|
@has_text_arg = nil
|
282
296
|
end
|
283
297
|
|
284
298
|
def to_s
|
285
299
|
prefix = @label ? '%' << @label << ':' : ''
|
286
|
-
return prefix << super
|
300
|
+
return( prefix << super )
|
287
301
|
end
|
288
|
-
|
289
302
|
end
|
290
303
|
|
291
|
-
=begin rdoc ANTLR3::AST::Wizard::
|
304
|
+
=begin rdoc ANTLR3::AST::Wizard::WildcardPattern
|
292
305
|
|
293
306
|
A simple tree node used to represent the operation "match any tree node type" in
|
294
307
|
a tree pattern. They are represented by '.' in tree pattern specifications.
|
295
308
|
|
296
309
|
=end
|
297
310
|
|
298
|
-
class
|
311
|
+
class WildcardPattern < Pattern; end
|
299
312
|
|
300
313
|
|
301
|
-
=begin rdoc ANTLR3::AST::Wizard::
|
314
|
+
=begin rdoc ANTLR3::AST::Wizard::PatternAdaptor
|
302
315
|
|
303
316
|
A customized TreeAdaptor used by AST::Wizards to build tree patterns.
|
304
317
|
|
305
318
|
=end
|
306
319
|
|
307
|
-
class
|
308
|
-
def create_with_payload!(payload)
|
309
|
-
return
|
320
|
+
class PatternAdaptor < CommonTreeAdaptor
|
321
|
+
def create_with_payload!( payload )
|
322
|
+
return Pattern.new( payload )
|
310
323
|
end
|
311
324
|
end
|
312
325
|
|
313
|
-
attr_accessor :
|
326
|
+
attr_accessor :token_scheme, :adaptor
|
314
327
|
|
315
|
-
def initialize(
|
316
|
-
@
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
end
|
323
|
-
|
324
|
-
def token_type(name)
|
325
|
-
@token_name_to_type_map[name]
|
328
|
+
def initialize( options = {} )
|
329
|
+
@token_scheme = options.fetch( :token_scheme ) do
|
330
|
+
TokenScheme.build( options[ :token_class ], options[ :tokens ] )
|
331
|
+
end
|
332
|
+
@adaptor = options.fetch( :adaptor ) do
|
333
|
+
CommonTreeAdaptor.new( @token_scheme.token_class )
|
334
|
+
end
|
326
335
|
end
|
327
336
|
|
328
|
-
def create(pattern)
|
329
|
-
|
330
|
-
parser = TreePatternParser.new(tokenizer, self, @adaptor)
|
331
|
-
return parser.pattern
|
337
|
+
def create( pattern )
|
338
|
+
PatternParser.parse( pattern, @token_scheme, @adaptor )
|
332
339
|
end
|
333
340
|
|
334
|
-
def index(tree)
|
335
|
-
|
336
|
-
|
337
|
-
|
341
|
+
def index( tree, map = {} )
|
342
|
+
tree or return( map )
|
343
|
+
type = @adaptor.type_of( tree )
|
344
|
+
elements = map[ type ] ||= []
|
345
|
+
elements << tree
|
346
|
+
@adaptor.each_child( tree ) { | child | index( child, map ) }
|
347
|
+
return( map )
|
338
348
|
end
|
339
349
|
|
340
|
-
def find(tree, what)
|
350
|
+
def find( tree, what )
|
341
351
|
case what
|
342
352
|
when Integer then find_token_type(tree, what)
|
343
|
-
when String
|
353
|
+
when String then find_pattern(tree, what)
|
354
|
+
when Symbol then find_token_type( tree, @token_scheme[ what ] )
|
344
355
|
else raise ArgumentError, "search subject must be a token type (integer) or a string"
|
345
356
|
end
|
346
357
|
end
|
347
358
|
|
348
359
|
def find_token_type(tree, type)
|
349
360
|
nodes = []
|
350
|
-
visit(tree, type)
|
351
|
-
nodes << t
|
352
|
-
end
|
361
|
+
visit( tree, type ) { | t, | nodes << t }
|
353
362
|
return nodes
|
354
363
|
end
|
355
364
|
|
356
365
|
def find_pattern(tree, pattern)
|
357
366
|
subtrees = []
|
358
|
-
|
359
|
-
|
360
|
-
pattern = parser.pattern()
|
361
|
-
|
362
|
-
return nil if pattern.nil? or pattern.flat_list? or
|
363
|
-
pattern.is_a?(WildcardTreePattern)
|
364
|
-
|
365
|
-
root_token_type = pattern.type
|
366
|
-
visit(tree, root_token_type) do |t, parent, child_index, label|
|
367
|
-
( parse! t, pattern, nil ) and subtrees << t
|
368
|
-
end
|
369
|
-
return subtrees
|
367
|
+
visit_pattern( tree, pattern ) { | t, | subtrees << t }
|
368
|
+
return( subtrees )
|
370
369
|
end
|
371
370
|
|
372
|
-
def visit(tree, what, &block)
|
371
|
+
def visit( tree, what = nil, &block )
|
372
|
+
block_given? or return enum_for( :visit, tree, what )
|
373
|
+
Symbol === what and what = @token_scheme[ what ]
|
373
374
|
case what
|
374
|
-
when
|
375
|
-
when
|
376
|
-
|
377
|
-
|
375
|
+
when nil then visit_all( tree, &block )
|
376
|
+
when Integer then visit_type( tree, nil, what, &block )
|
377
|
+
when String then visit_pattern( tree, what, &block )
|
378
|
+
else raise( ArgumentError, tidy( <<-'END', true ) )
|
379
|
+
| The 'what' filter argument must be a tree
|
380
|
+
| pattern (String) or a token type (Integer)
|
381
|
+
| -- got #{ what.inspect }
|
382
|
+
END
|
378
383
|
end
|
379
384
|
end
|
380
385
|
|
381
|
-
def
|
386
|
+
def visit_all( tree, parent = nil, &block )
|
387
|
+
index = @adaptor.child_index( tree )
|
388
|
+
yield( tree, parent, index, nil )
|
389
|
+
@adaptor.each_child( tree ) do | child |
|
390
|
+
visit_all( child, tree, &block )
|
391
|
+
end
|
392
|
+
end
|
393
|
+
|
394
|
+
def visit_type( tree, parent, type, &block )
|
382
395
|
tree.nil? and return(nil)
|
383
|
-
@adaptor.
|
384
|
-
@adaptor.
|
385
|
-
|
386
|
-
visit_type(child, tree,
|
396
|
+
index = @adaptor.child_index( tree )
|
397
|
+
@adaptor.type_of( tree ) == type and yield( tree, parent, index, nil )
|
398
|
+
@adaptor.each_child( tree ) do | child |
|
399
|
+
visit_type( child, tree, type, &block )
|
387
400
|
end
|
388
401
|
end
|
389
402
|
|
390
|
-
def visit_pattern(tree, pattern, &block)
|
391
|
-
|
392
|
-
parser = TreePatternParser.new(tokenizer, self, TreePatternTreeAdaptor.new)
|
393
|
-
pattern = parser.pattern()
|
394
|
-
|
395
|
-
return nil if pattern.nil? or pattern.flat_list? or
|
396
|
-
pattern.is_a?(WildcardTreePattern)
|
403
|
+
def visit_pattern( tree, pattern, &block )
|
404
|
+
pattern = Pattern.parse( pattern, @token_scheme )
|
397
405
|
|
398
|
-
|
406
|
+
if pattern.nil? or pattern.flat_list? or pattern.is_a?( WildcardPattern )
|
407
|
+
return( nil )
|
408
|
+
end
|
399
409
|
|
400
|
-
visit tree,
|
401
|
-
labels =
|
402
|
-
|
410
|
+
visit( tree, pattern.type ) do | tree, parent, child_index, labels |
|
411
|
+
labels = match!( tree, pattern ) and
|
412
|
+
yield( tree, parent, child_index, labels )
|
403
413
|
end
|
404
414
|
end
|
405
415
|
|
406
|
-
def
|
407
|
-
|
408
|
-
parser = TreePatternParser.new(tokenizer, self, TreePatternTreeAdaptor.new)
|
409
|
-
pattern = parser.pattern
|
416
|
+
def match( tree, pattern )
|
417
|
+
pattern = Pattern.parse( pattern, @token_scheme )
|
410
418
|
|
411
|
-
|
419
|
+
return( match!( tree, pattern ) )
|
412
420
|
end
|
413
421
|
|
414
|
-
def
|
422
|
+
def match!( tree, pattern, labels = {} )
|
415
423
|
tree.nil? || pattern.nil? and return false
|
416
|
-
unless pattern.is_a?
|
424
|
+
unless pattern.is_a? WildcardPattern
|
417
425
|
@adaptor.type_of(tree) == pattern.type or return false
|
418
426
|
pattern.has_text_arg && (@adaptor.text_of(tree) != pattern.text) and
|
419
427
|
return false
|
420
428
|
end
|
421
|
-
labels[pattern.label] = tree if labels && pattern.label
|
429
|
+
labels[ pattern.label ] = tree if labels && pattern.label
|
422
430
|
|
423
431
|
number_of_children = @adaptor.child_count(tree)
|
424
432
|
return false unless number_of_children == pattern.child_count
|
425
433
|
|
426
434
|
number_of_children.times do |index|
|
427
435
|
actual_child = @adaptor.child_of(tree, index)
|
428
|
-
pattern_child = pattern.child(index)
|
429
|
-
|
436
|
+
pattern_child = pattern.child( index )
|
437
|
+
|
438
|
+
return(false) unless match!( actual_child, pattern_child, labels )
|
430
439
|
end
|
431
440
|
|
432
|
-
return
|
433
|
-
end
|
434
|
-
|
435
|
-
def equals(tree_a, tree_b, adaptor = nil)
|
436
|
-
adaptor ||= @adaptor
|
437
|
-
return equals!(tree_a, tree_b, adaptor)
|
441
|
+
return labels
|
438
442
|
end
|
439
443
|
|
440
|
-
def equals
|
441
|
-
tree_a && tree_b or return false
|
444
|
+
def equals( tree_a, tree_b, adaptor = @adaptor )
|
445
|
+
tree_a && tree_b or return( false )
|
442
446
|
|
443
447
|
adaptor.type_of(tree_a) == adaptor.type_of(tree_b) or return false
|
444
448
|
adaptor.text_of(tree_a) == adaptor.text_of(tree_b) or return false
|
@@ -447,35 +451,43 @@ A customized TreeAdaptor used by AST::Wizards to build tree patterns.
|
|
447
451
|
child_count_b = adaptor.child_count(tree_b)
|
448
452
|
child_count_a == child_count_b or return false
|
449
453
|
|
450
|
-
child_count_a.times do |
|
451
|
-
child_a = adaptor.child_of(tree_a,
|
452
|
-
child_b = adaptor.child_of(tree_b,
|
453
|
-
equals
|
454
|
+
child_count_a.times do | i |
|
455
|
+
child_a = adaptor.child_of( tree_a, i )
|
456
|
+
child_b = adaptor.child_of( tree_b, i )
|
457
|
+
equals( child_a, child_b, adaptor ) or return false
|
454
458
|
end
|
455
459
|
return true
|
456
460
|
end
|
457
461
|
|
458
|
-
def index!(tree, m)
|
459
|
-
tree or return nil
|
460
|
-
type = @adaptor.type_of(tree)
|
461
|
-
elements = (m[type] ||= [])
|
462
|
-
elements << tree
|
463
|
-
@adaptor.child_count(tree).times do |i|
|
464
|
-
child = @adaptor.child_of(tree, i)
|
465
|
-
index!(child, m)
|
466
|
-
end
|
467
|
-
end
|
468
462
|
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
463
|
+
DOT_DOT_PATTERN = /.*[^\.]\\.{2}[^\.].*/
|
464
|
+
DOUBLE_ETC_PATTERN = /.*\.{3}\s+\.{3}.*/
|
465
|
+
|
466
|
+
def in_context?( tree, context )
|
467
|
+
case context
|
468
|
+
when DOT_DOT_PATTERN then raise ArgumentError, "invalid syntax: .."
|
469
|
+
when DOUBLE_ETC_PATTERN then raise ArgumentError, "invalid syntax: ... ..."
|
474
470
|
end
|
475
|
-
|
471
|
+
|
472
|
+
context = context.gsub(/([^\.\s])\.{3}([^\.])/, '\1 ... \2')
|
473
|
+
context.strip!
|
474
|
+
nodes = context.split(/\s+/)
|
475
|
+
|
476
|
+
while tree = @adaptor.parent( tree ) and node = nodes.pop
|
477
|
+
if node == '...'
|
478
|
+
node = nodes.pop or return( true )
|
479
|
+
tree = @adaptor.each_ancestor( tree ).find do | t |
|
480
|
+
@adaptor.type_name( t ) == node
|
481
|
+
end or return( false )
|
482
|
+
end
|
483
|
+
@adaptor.type_name( tree ) == node or return( false )
|
484
|
+
end
|
485
|
+
|
486
|
+
return( false ) if tree.nil? and not nodes.empty?
|
487
|
+
return true
|
476
488
|
end
|
477
489
|
|
478
|
-
private :
|
490
|
+
private :match!
|
479
491
|
end
|
480
492
|
end
|
481
493
|
end
|
data/lib/antlr3/util.rb
CHANGED
@@ -14,7 +14,13 @@ module_function
|
|
14
14
|
str
|
15
15
|
end
|
16
16
|
|
17
|
-
def
|
17
|
+
def parse_version( version_string )
|
18
|
+
version_string.split( '.' ).map! do | segment |
|
19
|
+
segment.to_i
|
20
|
+
end.freeze
|
21
|
+
end
|
22
|
+
|
23
|
+
def tidy( here_doc, flow = false )
|
18
24
|
here_doc.gsub!(/^ *\| ?/, '')
|
19
25
|
if flow
|
20
26
|
here_doc.strip!
|
data/lib/antlr3/version.rb
CHANGED
@@ -10,7 +10,7 @@ module ANTLR3
|
|
10
10
|
ANTLR_MAJOR_VERSION = 3
|
11
11
|
ANTLR_MINOR_VERSION = 2
|
12
12
|
ANTLR_PATCH_VERSION = 1
|
13
|
-
ANTLR_VERSION = [ANTLR_MAJOR_VERSION, ANTLR_MINOR_VERSION, ANTLR_PATCH_VERSION].freeze
|
13
|
+
ANTLR_VERSION = [ ANTLR_MAJOR_VERSION, ANTLR_MINOR_VERSION, ANTLR_PATCH_VERSION ].freeze
|
14
14
|
ANTLR_VERSION_STRING = ANTLR_VERSION.join('.')
|
15
15
|
ANTLR_VERSION_STRING.chomp!( '.0' ) # versioning drops minor version at 0
|
16
16
|
ANTLR_VERSION_STRING.freeze
|
@@ -19,9 +19,9 @@ module ANTLR3
|
|
19
19
|
# The version data for the current state the library itself
|
20
20
|
#
|
21
21
|
MAJOR_VERSION = 1
|
22
|
-
MINOR_VERSION =
|
23
|
-
PATCH_VERSION =
|
24
|
-
VERSION = [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
|
22
|
+
MINOR_VERSION = 3
|
23
|
+
PATCH_VERSION = 0
|
24
|
+
VERSION = [ MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION ]
|
25
25
|
VERSION_STRING = VERSION.join('.').freeze
|
26
26
|
|
27
27
|
end
|