antlr3 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ANTLR-LICENSE.txt +26 -0
- data/History.txt +66 -0
- data/README.txt +139 -0
- data/bin/antlr4ruby +33 -0
- data/java/RubyTarget.java +524 -0
- data/java/antlr-full-3.2.1.jar +0 -0
- data/lib/antlr3.rb +176 -0
- data/lib/antlr3/constants.rb +88 -0
- data/lib/antlr3/debug.rb +701 -0
- data/lib/antlr3/debug/event-hub.rb +210 -0
- data/lib/antlr3/debug/record-event-listener.rb +25 -0
- data/lib/antlr3/debug/rule-tracer.rb +55 -0
- data/lib/antlr3/debug/socket.rb +360 -0
- data/lib/antlr3/debug/trace-event-listener.rb +92 -0
- data/lib/antlr3/dfa.rb +247 -0
- data/lib/antlr3/dot.rb +174 -0
- data/lib/antlr3/error.rb +657 -0
- data/lib/antlr3/main.rb +561 -0
- data/lib/antlr3/modes/ast-builder.rb +41 -0
- data/lib/antlr3/modes/filter.rb +56 -0
- data/lib/antlr3/profile.rb +322 -0
- data/lib/antlr3/recognizers.rb +1280 -0
- data/lib/antlr3/streams.rb +985 -0
- data/lib/antlr3/streams/interactive.rb +91 -0
- data/lib/antlr3/streams/rewrite.rb +412 -0
- data/lib/antlr3/test/call-stack.rb +57 -0
- data/lib/antlr3/test/config.rb +23 -0
- data/lib/antlr3/test/core-extensions.rb +269 -0
- data/lib/antlr3/test/diff.rb +165 -0
- data/lib/antlr3/test/functional.rb +207 -0
- data/lib/antlr3/test/grammar.rb +371 -0
- data/lib/antlr3/token.rb +592 -0
- data/lib/antlr3/tree.rb +1415 -0
- data/lib/antlr3/tree/debug.rb +163 -0
- data/lib/antlr3/tree/visitor.rb +84 -0
- data/lib/antlr3/tree/wizard.rb +481 -0
- data/lib/antlr3/util.rb +149 -0
- data/lib/antlr3/version.rb +27 -0
- data/samples/ANTLRv3Grammar.g +621 -0
- data/samples/Cpp.g +749 -0
- data/templates/AST.stg +335 -0
- data/templates/ASTDbg.stg +40 -0
- data/templates/ASTParser.stg +153 -0
- data/templates/ASTTreeParser.stg +272 -0
- data/templates/Dbg.stg +192 -0
- data/templates/Ruby.stg +1514 -0
- data/test/functional/ast-output/auto-ast.rb +797 -0
- data/test/functional/ast-output/construction.rb +555 -0
- data/test/functional/ast-output/hetero-nodes.rb +753 -0
- data/test/functional/ast-output/rewrites.rb +1327 -0
- data/test/functional/ast-output/tree-rewrite.rb +1662 -0
- data/test/functional/debugging/debug-mode.rb +689 -0
- data/test/functional/debugging/profile-mode.rb +165 -0
- data/test/functional/debugging/rule-tracing.rb +74 -0
- data/test/functional/delegation/import.rb +379 -0
- data/test/functional/lexer/basic.rb +559 -0
- data/test/functional/lexer/filter-mode.rb +245 -0
- data/test/functional/lexer/nuances.rb +47 -0
- data/test/functional/lexer/properties.rb +104 -0
- data/test/functional/lexer/syn-pred.rb +32 -0
- data/test/functional/lexer/xml.rb +206 -0
- data/test/functional/main/main-scripts.rb +245 -0
- data/test/functional/parser/actions.rb +224 -0
- data/test/functional/parser/backtracking.rb +244 -0
- data/test/functional/parser/basic.rb +282 -0
- data/test/functional/parser/calc.rb +98 -0
- data/test/functional/parser/ll-star.rb +143 -0
- data/test/functional/parser/nuances.rb +165 -0
- data/test/functional/parser/predicates.rb +103 -0
- data/test/functional/parser/properties.rb +242 -0
- data/test/functional/parser/rule-methods.rb +132 -0
- data/test/functional/parser/scopes.rb +274 -0
- data/test/functional/token-rewrite/basic.rb +318 -0
- data/test/functional/token-rewrite/via-parser.rb +100 -0
- data/test/functional/tree-parser/basic.rb +750 -0
- data/test/unit/sample-input/file-stream-1 +2 -0
- data/test/unit/sample-input/teststreams.input2 +2 -0
- data/test/unit/test-dfa.rb +52 -0
- data/test/unit/test-exceptions.rb +44 -0
- data/test/unit/test-recognizers.rb +55 -0
- data/test/unit/test-scheme.rb +62 -0
- data/test/unit/test-streams.rb +459 -0
- data/test/unit/test-tree-wizard.rb +535 -0
- data/test/unit/test-trees.rb +854 -0
- metadata +205 -0
@@ -0,0 +1,163 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
module ANTLR3
|
5
|
+
module Debug
|
6
|
+
=begin rdoc ANTLR3::Debug::TreeAdaptor
|
7
|
+
|
8
|
+
Adds debugging event hooks to TreeAdaptor objects
|
9
|
+
|
10
|
+
=end
|
11
|
+
module TreeAdaptor
|
12
|
+
def self.wrap(adaptor, debug_listener = nil)
|
13
|
+
adaptor.extend(self)
|
14
|
+
adaptor.debug_listener = debug_listener
|
15
|
+
return(adaptor)
|
16
|
+
end
|
17
|
+
|
18
|
+
attr_accessor :debug_listener
|
19
|
+
|
20
|
+
def create_with_payload!(payload)
|
21
|
+
node = super
|
22
|
+
@debug_listener.create_node(node, payload)
|
23
|
+
return node
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_from_token!(token_type, from_token, text = nil)
|
27
|
+
node = super
|
28
|
+
@debug_listener.create_node(node)
|
29
|
+
return node
|
30
|
+
end
|
31
|
+
|
32
|
+
def create_from_type!(token_type, text)
|
33
|
+
node = super
|
34
|
+
@debug_listener.create_node(node)
|
35
|
+
return node
|
36
|
+
end
|
37
|
+
|
38
|
+
def create_error_node!(input, start, stop, exc)
|
39
|
+
node = super
|
40
|
+
node.nil? or @debug_listener.error_node(node)
|
41
|
+
return node
|
42
|
+
end
|
43
|
+
|
44
|
+
def copy_tree(tree)
|
45
|
+
t = super
|
46
|
+
simulate_tree_construction(t)
|
47
|
+
return t
|
48
|
+
end
|
49
|
+
|
50
|
+
def simulate_tree_construction(tree)
|
51
|
+
@debug_listener.create_node(tree)
|
52
|
+
child_count(tree).times do |i|
|
53
|
+
child = self.child_of(tree, i)
|
54
|
+
simulate_tree_construction(child)
|
55
|
+
@debug_listener.add_child(tree, child)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def copy_node(tree_node)
|
60
|
+
duplicate = super
|
61
|
+
@debug_listener.create_node duplicate
|
62
|
+
return duplicate
|
63
|
+
end
|
64
|
+
|
65
|
+
def create_flat_list!
|
66
|
+
node = super
|
67
|
+
@debug_listener.flat_node(node)
|
68
|
+
return node
|
69
|
+
end
|
70
|
+
|
71
|
+
def add_child(tree, child)
|
72
|
+
case child
|
73
|
+
when Token
|
74
|
+
node = create_with_payload!(child)
|
75
|
+
add_child(tree, node)
|
76
|
+
else
|
77
|
+
tree.nil? || child.nil? and return
|
78
|
+
super(tree, child)
|
79
|
+
@debug_listener.add_child(tree, child)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def become_root(new_root, old_root)
|
84
|
+
case new_root
|
85
|
+
when Token
|
86
|
+
n = create_with_payload!(new_root)
|
87
|
+
super(n, old_root)
|
88
|
+
else
|
89
|
+
n = super(new_root, old_root)
|
90
|
+
end
|
91
|
+
@debug_listener.become_root(new_root, old_root)
|
92
|
+
return n
|
93
|
+
end
|
94
|
+
|
95
|
+
def set_token_boundaries(tree, start_token, stop_token)
|
96
|
+
super(tree, start_token, stop_token)
|
97
|
+
return unless tree && start_token && stop_token
|
98
|
+
@debug_listener.set_token_boundaries(tree,
|
99
|
+
start_token.token_index, stop_token.token_index)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
=begin rdoc ANTLR3::Debug::TreeNodeStream
|
104
|
+
|
105
|
+
A module that wraps token stream methods with debugging event code. A debuggable
|
106
|
+
parser will <tt>extend</tt> its input stream with this module if the stream is
|
107
|
+
not already a Debug::TreeNodeStream.
|
108
|
+
|
109
|
+
=end
|
110
|
+
class TreeNodeStream
|
111
|
+
|
112
|
+
def self.wrap(stream, debug_listener = nil)
|
113
|
+
stream.extend(self)
|
114
|
+
stream.debug_listener ||= debug_listener
|
115
|
+
end
|
116
|
+
attr_accessor :debug_listener
|
117
|
+
|
118
|
+
def consume
|
119
|
+
node = @input >> 1
|
120
|
+
super
|
121
|
+
@debug_listener.consume_node(node)
|
122
|
+
end
|
123
|
+
|
124
|
+
def look(i = 1)
|
125
|
+
node = super
|
126
|
+
id = @adaptor.unique_id(node)
|
127
|
+
text = @adaptor.text_of(node)
|
128
|
+
type = @adaptor.type_of(node)
|
129
|
+
@debug_listener.look(i, node)
|
130
|
+
return(node)
|
131
|
+
end
|
132
|
+
|
133
|
+
def peek(i = 1)
|
134
|
+
node = self >> 1
|
135
|
+
id = @adaptor.unique_id(node)
|
136
|
+
text = @adaptor.text_of(node)
|
137
|
+
type = @adaptor.type_of(node)
|
138
|
+
@debug_listener.look(i, node)
|
139
|
+
return(type)
|
140
|
+
end
|
141
|
+
|
142
|
+
def mark
|
143
|
+
@last_marker = super
|
144
|
+
@debug_listener.mark(@last_marker)
|
145
|
+
return(@last_marker)
|
146
|
+
end
|
147
|
+
|
148
|
+
def rewind(marker = nil)
|
149
|
+
@debug_listener.rewind(marker)
|
150
|
+
super(marker || @last_marker)
|
151
|
+
end
|
152
|
+
|
153
|
+
=begin This actually differs with reset in CommonTreeNodeStream -- why is this one blank?
|
154
|
+
def reset
|
155
|
+
# do nothing
|
156
|
+
end
|
157
|
+
=end
|
158
|
+
|
159
|
+
end
|
160
|
+
|
161
|
+
|
162
|
+
end
|
163
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
=begin LICENSE
|
5
|
+
|
6
|
+
[The "BSD licence"]
|
7
|
+
Copyright (c) 2009 Kyle Yetter
|
8
|
+
All rights reserved.
|
9
|
+
|
10
|
+
Redistribution and use in source and binary forms, with or without
|
11
|
+
modification, are permitted provided that the following conditions
|
12
|
+
are met:
|
13
|
+
|
14
|
+
1. Redistributions of source code must retain the above copyright
|
15
|
+
notice, this list of conditions and the following disclaimer.
|
16
|
+
2. Redistributions in binary form must reproduce the above copyright
|
17
|
+
notice, this list of conditions and the following disclaimer in the
|
18
|
+
documentation and/or other materials provided with the distribution.
|
19
|
+
3. The name of the author may not be used to endorse or promote products
|
20
|
+
derived from this software without specific prior written permission.
|
21
|
+
|
22
|
+
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
23
|
+
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
24
|
+
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
25
|
+
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
26
|
+
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
27
|
+
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
28
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
29
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
30
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
31
|
+
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
32
|
+
|
33
|
+
=end
|
34
|
+
|
35
|
+
module ANTLR3
|
36
|
+
module AST
|
37
|
+
|
38
|
+
=begin rdoc ANTLR3::AST::Visitor
|
39
|
+
|
40
|
+
AST::Visitor is an extra utility class for working with tree objects. Visitor
|
41
|
+
objects are similar to plain vanilla iterators, but provide two action hooks,
|
42
|
+
instead a standard single iteration block. The <tt>visit(tree)</tt> method walks
|
43
|
+
through each node of the tree (top-down left-to-right). If +pre_action+ is
|
44
|
+
defined, a node is yielded to the block when it has been initially entered. If
|
45
|
+
+post_action+ is defined, a node is yielded to the block after all of its
|
46
|
+
children have been visited.
|
47
|
+
|
48
|
+
=end
|
49
|
+
|
50
|
+
class Visitor
|
51
|
+
def initialize(adaptor = nil)
|
52
|
+
@adaptor = adaptor || CommonTreeAdaptor.new()
|
53
|
+
@pre_action = nil
|
54
|
+
@post_action = nil
|
55
|
+
block_given? and yield(self)
|
56
|
+
end
|
57
|
+
|
58
|
+
def pre_action(&block)
|
59
|
+
block_given? and @pre_action = block
|
60
|
+
return @pre_action
|
61
|
+
end
|
62
|
+
|
63
|
+
def post_action(&block)
|
64
|
+
block_given? and @post_action = block
|
65
|
+
return @post_action
|
66
|
+
end
|
67
|
+
|
68
|
+
def visit(tree, pre_action = nil, post_action = nil)
|
69
|
+
flat = @adaptor.flat_list?(tree)
|
70
|
+
before = pre_action || @pre_action
|
71
|
+
after = post_action || @post_action
|
72
|
+
|
73
|
+
tree = before.call(tree) unless before.nil? or flat
|
74
|
+
@adaptor.child_count(tree).times do |index|
|
75
|
+
child = @adaptor.child_of(tree, index)
|
76
|
+
visit(child, pre_action, post_action)
|
77
|
+
end
|
78
|
+
tree = after.call(tree) unless after.nil? or flat
|
79
|
+
|
80
|
+
return tree
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,481 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
=begin LICENSE
|
5
|
+
|
6
|
+
[The "BSD licence"]
|
7
|
+
Copyright (c) 2009 Kyle Yetter
|
8
|
+
All rights reserved.
|
9
|
+
|
10
|
+
Redistribution and use in source and binary forms, with or without
|
11
|
+
modification, are permitted provided that the following conditions
|
12
|
+
are met:
|
13
|
+
|
14
|
+
1. Redistributions of source code must retain the above copyright
|
15
|
+
notice, this list of conditions and the following disclaimer.
|
16
|
+
2. Redistributions in binary form must reproduce the above copyright
|
17
|
+
notice, this list of conditions and the following disclaimer in the
|
18
|
+
documentation and/or other materials provided with the distribution.
|
19
|
+
3. The name of the author may not be used to endorse or promote products
|
20
|
+
derived from this software without specific prior written permission.
|
21
|
+
|
22
|
+
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
23
|
+
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
24
|
+
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
25
|
+
IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
26
|
+
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
27
|
+
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
28
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
29
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
30
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
31
|
+
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
32
|
+
|
33
|
+
=end
|
34
|
+
|
35
|
+
require 'antlr3'
|
36
|
+
|
37
|
+
module ANTLR3
|
38
|
+
module AST
|
39
|
+
|
40
|
+
=begin rdoc ANTLR3::AST::Wizard
|
41
|
+
|
42
|
+
AST::Wizard is an extra utility class that allows quick creation of AST objects
|
43
|
+
using definitions writing in ANTLR-style tree definition syntax. It can also
|
44
|
+
define <i>tree patterns</i>, objects that are conceptually similar to regular
|
45
|
+
expressions. Patterns allow a simple method for recursively searching through an
|
46
|
+
AST for a particular node structure. These features make tree wizards useful
|
47
|
+
while testing and debugging AST constructing parsers and tree parsers. This
|
48
|
+
library has been ported to Ruby directly from the ANTLR Python runtime API.
|
49
|
+
|
50
|
+
See
|
51
|
+
http://www.antlr.org/wiki/display/~admin/2007/07/02/Exploring+Concept+of+TreeWizard
|
52
|
+
for more background on the concept of a tree wizard.
|
53
|
+
|
54
|
+
== Usage
|
55
|
+
|
56
|
+
# setting up and creating a tree wizard
|
57
|
+
token_names = Array.new(4, '') + %w(VAR NUMBER EQ PLUS MINUS MULT DIV)
|
58
|
+
adaptor = ANTLR3::AST::CommonTreeAdaptor.new
|
59
|
+
wizard = ANTLR3::AST::Wizard.new(adaptor, token_names)
|
60
|
+
|
61
|
+
# building trees
|
62
|
+
lone_node = wizard.create "VAR[x]" # => x
|
63
|
+
lone_node.type # => 4 # = VAR
|
64
|
+
lone_node.text # => "x"
|
65
|
+
|
66
|
+
expression_node = wizard.create "(MINUS VAR NUMBER)"
|
67
|
+
# => (MINUS VAR NUMBER)
|
68
|
+
statement_node = wizard.create "(EQ[=] VAR[x] (PLUS[+] NUMBER[1] NUMBER[2]))"
|
69
|
+
# => (= x (+ 1 2))
|
70
|
+
deep_node = wizard.create(<<-TREE)
|
71
|
+
(MULT[*] NUMBER[1]
|
72
|
+
(MINUS[-]
|
73
|
+
(MULT[*] NUMBER[3] VAR[x])
|
74
|
+
(DIV[/] VAR[y] NUMBER[3.14])
|
75
|
+
(MULT[*] NUMBER[4] VAR[z])
|
76
|
+
)
|
77
|
+
)
|
78
|
+
TREE
|
79
|
+
# => (* 1 (- (* 3 x) (/ y 3.14) (* 4 z))
|
80
|
+
|
81
|
+
bad_tree_syntax = wizard.create "(+ 1 2)"
|
82
|
+
# => nil - invalid node names
|
83
|
+
|
84
|
+
# test whether a tree matches a pattern
|
85
|
+
wizard.parse(expression_node, '(MINUS VAR .)') # => true
|
86
|
+
wizard.parse(lone_node, 'NUMBER NUMBER') # => false
|
87
|
+
|
88
|
+
# extract nodes matching a pattern
|
89
|
+
wizard.find(statement_node, '(PLUS . .)')
|
90
|
+
# => [(+ 1 2)]
|
91
|
+
wizard.find(deep_node, 4) # where 4 is the value of type VAR
|
92
|
+
# => [x, y, z]
|
93
|
+
|
94
|
+
# iterate through the tree and extract nodes with pattern labels
|
95
|
+
wizard.visit(deep_node, '(MULT %n:NUMBER %v:.)') do |node, parent, local_index, labels|
|
96
|
+
printf "n = %p\n, v = %p\n", labels['n'], labels['v']
|
97
|
+
end
|
98
|
+
# => prints out:
|
99
|
+
# n = 3, v = x
|
100
|
+
# n = 4, v = z
|
101
|
+
|
102
|
+
== Tree Construction Syntax
|
103
|
+
|
104
|
+
Simple Token Node: TK
|
105
|
+
Token Node With Text: TK[text]
|
106
|
+
Flat Node List: (nil TK1 TK2)
|
107
|
+
General Node: (RT TK1 TK2)
|
108
|
+
Complex Nested Node: (RT (SUB1[sometext] TK1) TK2 (SUB2 TK3 TK4[moretext]))
|
109
|
+
|
110
|
+
=== Additional Syntax for Tree Matching Patterns
|
111
|
+
|
112
|
+
Match Any Token Node: .
|
113
|
+
Label a Node: %name:TK
|
114
|
+
|
115
|
+
=end
|
116
|
+
|
117
|
+
class Wizard
|
118
|
+
|
119
|
+
include ANTLR3::Constants
|
120
|
+
|
121
|
+
=begin rdoc ANTLR3::AST::Wizard::TreePatternLexer
|
122
|
+
|
123
|
+
A class that is used internally by AST::Wizard to tokenize tree patterns
|
124
|
+
|
125
|
+
=end
|
126
|
+
|
127
|
+
class TreePatternLexer
|
128
|
+
include ANTLR3::Constants
|
129
|
+
|
130
|
+
autoload :StringScanner, 'strscan'
|
131
|
+
|
132
|
+
PATTERNS = [
|
133
|
+
[:space, /\s+/],
|
134
|
+
[:identifier, /[a-z_]\w*/i],
|
135
|
+
[:open, /\(/],
|
136
|
+
[:close, /\)/],
|
137
|
+
[:percent, /%/],
|
138
|
+
[:colon, /:/],
|
139
|
+
[:dot, /\./],
|
140
|
+
[:argument, /\[((?:[^\[\]\\]|\\\[|\\\]|\\.)*?)\]/]
|
141
|
+
]
|
142
|
+
|
143
|
+
attr_reader :text, :error, :pattern
|
144
|
+
def initialize(pattern)
|
145
|
+
@pattern = pattern.to_s
|
146
|
+
@scanner = StringScanner.new(pattern)
|
147
|
+
@text = ''
|
148
|
+
@error = false
|
149
|
+
end
|
150
|
+
|
151
|
+
def next_token
|
152
|
+
begin
|
153
|
+
@scanner.eos? and return EOF
|
154
|
+
|
155
|
+
type, = PATTERNS.find do |type, pattern|
|
156
|
+
@scanner.scan(pattern)
|
157
|
+
end
|
158
|
+
|
159
|
+
case type
|
160
|
+
when nil
|
161
|
+
type, @text, @error = EOF, '', true
|
162
|
+
break
|
163
|
+
when :identifier then @text = @scanner.matched
|
164
|
+
when :argument
|
165
|
+
# remove escapes from \] sequences in the text argument
|
166
|
+
(@text = @scanner[1]).gsub!(/\\(?=[\[\]])/, '')
|
167
|
+
end
|
168
|
+
end while type == :space
|
169
|
+
|
170
|
+
return type
|
171
|
+
end
|
172
|
+
|
173
|
+
alias error? error
|
174
|
+
end
|
175
|
+
|
176
|
+
|
177
|
+
=begin rdoc ANTLR3::AST::Wizard::TreePatternParser
|
178
|
+
|
179
|
+
A class that is used internally by AST::Wizard to construct AST tree objects
|
180
|
+
from a tokenized tree pattern
|
181
|
+
|
182
|
+
=end
|
183
|
+
|
184
|
+
class TreePatternParser
|
185
|
+
include ANTLR3::Constants
|
186
|
+
|
187
|
+
def initialize(tokenizer, wizard, adaptor)
|
188
|
+
@tokenizer = tokenizer
|
189
|
+
@wizard = wizard
|
190
|
+
@adaptor = adaptor
|
191
|
+
@token_type = tokenizer.next_token
|
192
|
+
end
|
193
|
+
|
194
|
+
def pattern
|
195
|
+
case @token_type
|
196
|
+
when :open then return parse_tree
|
197
|
+
when :identifier
|
198
|
+
node = parse_node
|
199
|
+
@token_type == EOF and return node
|
200
|
+
return nil
|
201
|
+
end
|
202
|
+
return nil
|
203
|
+
end
|
204
|
+
|
205
|
+
CONTINUE_TYPES = [:open, :identifier, :percent, :dot]
|
206
|
+
|
207
|
+
def parse_tree
|
208
|
+
@token_type != :open and return nil
|
209
|
+
@token_type = @tokenizer.next_token
|
210
|
+
root = parse_node or return nil
|
211
|
+
|
212
|
+
loop do
|
213
|
+
case @token_type
|
214
|
+
when :open
|
215
|
+
subtree = parse_tree()
|
216
|
+
@adaptor.add_child(root, subtree)
|
217
|
+
when :identifier, :percent, :dot
|
218
|
+
child = parse_node or return nil
|
219
|
+
@adaptor.add_child(root, child)
|
220
|
+
else break
|
221
|
+
end
|
222
|
+
end
|
223
|
+
@token_type == :close or return nil
|
224
|
+
@token_type = @tokenizer.next_token
|
225
|
+
return root
|
226
|
+
end
|
227
|
+
|
228
|
+
def parse_node
|
229
|
+
label = nil
|
230
|
+
if @token_type == :percent
|
231
|
+
(@token_type = @tokenizer.next_token) == :identifier or return nil
|
232
|
+
label = @tokenizer.text
|
233
|
+
(@token_type = @tokenizer.next_token) == :colon or return nil
|
234
|
+
@token_type = @tokenizer.next_token
|
235
|
+
end
|
236
|
+
|
237
|
+
if @token_type == :dot
|
238
|
+
@token_type = @tokenizer.next_token
|
239
|
+
wildcard_payload = CommonToken.create(:type => 0, :text => '.')
|
240
|
+
node = WildcardTreePattern.new(wildcard_payload)
|
241
|
+
label and node.label = label
|
242
|
+
return node
|
243
|
+
end
|
244
|
+
|
245
|
+
@token_type == :identifier or return nil
|
246
|
+
token_name = @tokenizer.text
|
247
|
+
@token_type = @tokenizer.next_token
|
248
|
+
token_name == 'nil' and return @adaptor.create_flat_list!
|
249
|
+
|
250
|
+
text = token_name
|
251
|
+
arg = nil
|
252
|
+
if @token_type == :argument
|
253
|
+
arg = @tokenizer.text
|
254
|
+
text = arg
|
255
|
+
@token_type = @tokenizer.next_token
|
256
|
+
end
|
257
|
+
|
258
|
+
tree_node_type = @wizard.token_type(token_name)
|
259
|
+
tree_node_type == INVALID_TOKEN_TYPE and return nil
|
260
|
+
|
261
|
+
node = @adaptor.create_from_type!(tree_node_type, text)
|
262
|
+
label && node.is_a?(TreePattern) and node.label = label
|
263
|
+
arg && node.is_a?(TreePattern) and node.has_text_arg = true
|
264
|
+
return node
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
|
269
|
+
=begin rdoc ANTLR3::AST::Wizard::TreePattern
|
270
|
+
|
271
|
+
A simple tree class that represents the skeletal structure of tree. It is used
|
272
|
+
to validate tree structures as well as to extract nodes that match the pattern.
|
273
|
+
|
274
|
+
=end
|
275
|
+
|
276
|
+
class TreePattern < CommonTree
|
277
|
+
attr_accessor :label, :has_text_arg
|
278
|
+
def initialize(payload)
|
279
|
+
super(payload)
|
280
|
+
@label = nil
|
281
|
+
@has_text_arg = nil
|
282
|
+
end
|
283
|
+
|
284
|
+
def to_s
|
285
|
+
prefix = @label ? '%' << @label << ':' : ''
|
286
|
+
return prefix << super()
|
287
|
+
end
|
288
|
+
|
289
|
+
end
|
290
|
+
|
291
|
+
=begin rdoc ANTLR3::AST::Wizard::WildcardTreePattern
|
292
|
+
|
293
|
+
A simple tree node used to represent the operation "match any tree node type" in
|
294
|
+
a tree pattern. They are represented by '.' in tree pattern specifications.
|
295
|
+
|
296
|
+
=end
|
297
|
+
|
298
|
+
class WildcardTreePattern < TreePattern; end
|
299
|
+
|
300
|
+
|
301
|
+
=begin rdoc ANTLR3::AST::Wizard::TreePatternTreeAdaptor
|
302
|
+
|
303
|
+
A customized TreeAdaptor used by AST::Wizards to build tree patterns.
|
304
|
+
|
305
|
+
=end
|
306
|
+
|
307
|
+
class TreePatternTreeAdaptor < CommonTreeAdaptor
|
308
|
+
def create_with_payload!(payload)
|
309
|
+
return TreePattern.new(payload)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
attr_accessor :token_name_to_type_map, :adaptor
|
314
|
+
|
315
|
+
def initialize(adaptor = CommonTreeAdaptor.new, token_names_or_type_map = {})
|
316
|
+
@adaptor = adaptor
|
317
|
+
@token_name_to_type_map =
|
318
|
+
case token_names_or_type_map
|
319
|
+
when Array, nil then compute_token_types(token_names_or_type_map)
|
320
|
+
else token_names_or_type_map
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
def token_type(name)
|
325
|
+
@token_name_to_type_map[name]
|
326
|
+
end
|
327
|
+
|
328
|
+
def create(pattern)
|
329
|
+
tokenizer = TreePatternLexer.new(pattern)
|
330
|
+
parser = TreePatternParser.new(tokenizer, self, @adaptor)
|
331
|
+
return parser.pattern
|
332
|
+
end
|
333
|
+
|
334
|
+
def index(tree)
|
335
|
+
m = {}
|
336
|
+
index!(tree, m)
|
337
|
+
return(m)
|
338
|
+
end
|
339
|
+
|
340
|
+
def find(tree, what)
|
341
|
+
case what
|
342
|
+
when Integer then find_token_type(tree, what)
|
343
|
+
when String then find_pattern(tree, what)
|
344
|
+
else raise ArgumentError, "search subject must be a token type (integer) or a string"
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
def find_token_type(tree, type)
|
349
|
+
nodes = []
|
350
|
+
visit(tree, type) do |t, parent, child_index, labels|
|
351
|
+
nodes << t
|
352
|
+
end
|
353
|
+
return nodes
|
354
|
+
end
|
355
|
+
|
356
|
+
def find_pattern(tree, pattern)
|
357
|
+
subtrees = []
|
358
|
+
tokenizer = TreePatternLexer.new(pattern)
|
359
|
+
parser = TreePatternParser.new(tokenizer, self, TreePatternTreeAdaptor.new)
|
360
|
+
pattern = parser.pattern()
|
361
|
+
|
362
|
+
return nil if pattern.nil? or pattern.flat_list? or
|
363
|
+
pattern.is_a?(WildcardTreePattern)
|
364
|
+
|
365
|
+
root_token_type = pattern.type
|
366
|
+
visit(tree, root_token_type) do |t, parent, child_index, label|
|
367
|
+
( parse! t, pattern, nil ) and subtrees << t
|
368
|
+
end
|
369
|
+
return subtrees
|
370
|
+
end
|
371
|
+
|
372
|
+
def visit(tree, what, &block)
|
373
|
+
case what
|
374
|
+
when Integer then self.visit_type(tree, nil, 0, what, &block)
|
375
|
+
when String then self.visit_pattern(tree, what, &block)
|
376
|
+
else raise ArgumentError,
|
377
|
+
"the 'what' filter argument must be a tree pattern (String) or a token type (Integer) -- got #{what.inspect}"
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
def visit_type(tree, parent, child_index, type, &block)
|
382
|
+
tree.nil? and return(nil)
|
383
|
+
@adaptor.type_of(tree) == type and yield(tree, parent, child_index, nil)
|
384
|
+
@adaptor.child_count(tree).times do |i|
|
385
|
+
child = @adaptor.child_of(tree, i)
|
386
|
+
visit_type(child, tree, i, type, &block)
|
387
|
+
end
|
388
|
+
end
|
389
|
+
|
390
|
+
def visit_pattern(tree, pattern, &block)
|
391
|
+
tokenizer = TreePatternLexer.new(pattern)
|
392
|
+
parser = TreePatternParser.new(tokenizer, self, TreePatternTreeAdaptor.new)
|
393
|
+
pattern = parser.pattern()
|
394
|
+
|
395
|
+
return nil if pattern.nil? or pattern.flat_list? or
|
396
|
+
pattern.is_a?(WildcardTreePattern)
|
397
|
+
|
398
|
+
root_token_type = pattern.type
|
399
|
+
|
400
|
+
visit tree, root_token_type do |tree, parent, child_index, labels|
|
401
|
+
labels = {}
|
402
|
+
( parse! tree, pattern, labels ) and yield(tree, parent, child_index, labels)
|
403
|
+
end
|
404
|
+
end
|
405
|
+
|
406
|
+
def parse(tree, pattern, labels = nil)
|
407
|
+
tokenizer = TreePatternLexer.new(pattern)
|
408
|
+
parser = TreePatternParser.new(tokenizer, self, TreePatternTreeAdaptor.new)
|
409
|
+
pattern = parser.pattern
|
410
|
+
|
411
|
+
parse! tree, pattern, labels
|
412
|
+
end
|
413
|
+
|
414
|
+
def parse!(tree, pattern, labels)
|
415
|
+
tree.nil? || pattern.nil? and return false
|
416
|
+
unless pattern.is_a? WildcardTreePattern
|
417
|
+
@adaptor.type_of(tree) == pattern.type or return false
|
418
|
+
pattern.has_text_arg && (@adaptor.text_of(tree) != pattern.text) and
|
419
|
+
return false
|
420
|
+
end
|
421
|
+
labels[pattern.label] = tree if labels && pattern.label
|
422
|
+
|
423
|
+
number_of_children = @adaptor.child_count(tree)
|
424
|
+
return false unless number_of_children == pattern.child_count
|
425
|
+
|
426
|
+
number_of_children.times do |index|
|
427
|
+
actual_child = @adaptor.child_of(tree, index)
|
428
|
+
pattern_child = pattern.child(index)
|
429
|
+
return false unless parse! actual_child, pattern_child, labels
|
430
|
+
end
|
431
|
+
|
432
|
+
return true
|
433
|
+
end
|
434
|
+
|
435
|
+
def equals(tree_a, tree_b, adaptor = nil)
|
436
|
+
adaptor ||= @adaptor
|
437
|
+
return equals!(tree_a, tree_b, adaptor)
|
438
|
+
end
|
439
|
+
|
440
|
+
def equals! tree_a, tree_b, adaptor
|
441
|
+
tree_a && tree_b or return false # -> if either argument is nil, return false
|
442
|
+
|
443
|
+
adaptor.type_of(tree_a) == adaptor.type_of(tree_b) or return false
|
444
|
+
adaptor.text_of(tree_a) == adaptor.text_of(tree_b) or return false
|
445
|
+
|
446
|
+
child_count_a = adaptor.child_count(tree_a)
|
447
|
+
child_count_b = adaptor.child_count(tree_b)
|
448
|
+
child_count_a == child_count_b or return false
|
449
|
+
|
450
|
+
child_count_a.times do |index|
|
451
|
+
child_a = adaptor.child_of(tree_a, index)
|
452
|
+
child_b = adaptor.child_of(tree_b, index)
|
453
|
+
equals!(child_a, child_b, adaptor) or return false
|
454
|
+
end
|
455
|
+
return true
|
456
|
+
end
|
457
|
+
|
458
|
+
def index!(tree, m)
|
459
|
+
tree or return nil
|
460
|
+
type = @adaptor.type_of(tree)
|
461
|
+
elements = (m[type] ||= [])
|
462
|
+
elements << tree
|
463
|
+
@adaptor.child_count(tree).times do |i|
|
464
|
+
child = @adaptor.child_of(tree, i)
|
465
|
+
index!(child, m)
|
466
|
+
end
|
467
|
+
end
|
468
|
+
|
469
|
+
def compute_token_types(token_names)
|
470
|
+
token_names or return({})
|
471
|
+
map = Hash.new(INVALID_TOKEN_TYPE)
|
472
|
+
token_names.each_with_index do |name, index|
|
473
|
+
map[name] = index
|
474
|
+
end
|
475
|
+
return map
|
476
|
+
end
|
477
|
+
|
478
|
+
private :parse!, :index!, :equals!
|
479
|
+
end
|
480
|
+
end
|
481
|
+
end
|