antlr3 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ANTLR-LICENSE.txt +26 -0
- data/History.txt +66 -0
- data/README.txt +139 -0
- data/bin/antlr4ruby +33 -0
- data/java/RubyTarget.java +524 -0
- data/java/antlr-full-3.2.1.jar +0 -0
- data/lib/antlr3.rb +176 -0
- data/lib/antlr3/constants.rb +88 -0
- data/lib/antlr3/debug.rb +701 -0
- data/lib/antlr3/debug/event-hub.rb +210 -0
- data/lib/antlr3/debug/record-event-listener.rb +25 -0
- data/lib/antlr3/debug/rule-tracer.rb +55 -0
- data/lib/antlr3/debug/socket.rb +360 -0
- data/lib/antlr3/debug/trace-event-listener.rb +92 -0
- data/lib/antlr3/dfa.rb +247 -0
- data/lib/antlr3/dot.rb +174 -0
- data/lib/antlr3/error.rb +657 -0
- data/lib/antlr3/main.rb +561 -0
- data/lib/antlr3/modes/ast-builder.rb +41 -0
- data/lib/antlr3/modes/filter.rb +56 -0
- data/lib/antlr3/profile.rb +322 -0
- data/lib/antlr3/recognizers.rb +1280 -0
- data/lib/antlr3/streams.rb +985 -0
- data/lib/antlr3/streams/interactive.rb +91 -0
- data/lib/antlr3/streams/rewrite.rb +412 -0
- data/lib/antlr3/test/call-stack.rb +57 -0
- data/lib/antlr3/test/config.rb +23 -0
- data/lib/antlr3/test/core-extensions.rb +269 -0
- data/lib/antlr3/test/diff.rb +165 -0
- data/lib/antlr3/test/functional.rb +207 -0
- data/lib/antlr3/test/grammar.rb +371 -0
- data/lib/antlr3/token.rb +592 -0
- data/lib/antlr3/tree.rb +1415 -0
- data/lib/antlr3/tree/debug.rb +163 -0
- data/lib/antlr3/tree/visitor.rb +84 -0
- data/lib/antlr3/tree/wizard.rb +481 -0
- data/lib/antlr3/util.rb +149 -0
- data/lib/antlr3/version.rb +27 -0
- data/samples/ANTLRv3Grammar.g +621 -0
- data/samples/Cpp.g +749 -0
- data/templates/AST.stg +335 -0
- data/templates/ASTDbg.stg +40 -0
- data/templates/ASTParser.stg +153 -0
- data/templates/ASTTreeParser.stg +272 -0
- data/templates/Dbg.stg +192 -0
- data/templates/Ruby.stg +1514 -0
- data/test/functional/ast-output/auto-ast.rb +797 -0
- data/test/functional/ast-output/construction.rb +555 -0
- data/test/functional/ast-output/hetero-nodes.rb +753 -0
- data/test/functional/ast-output/rewrites.rb +1327 -0
- data/test/functional/ast-output/tree-rewrite.rb +1662 -0
- data/test/functional/debugging/debug-mode.rb +689 -0
- data/test/functional/debugging/profile-mode.rb +165 -0
- data/test/functional/debugging/rule-tracing.rb +74 -0
- data/test/functional/delegation/import.rb +379 -0
- data/test/functional/lexer/basic.rb +559 -0
- data/test/functional/lexer/filter-mode.rb +245 -0
- data/test/functional/lexer/nuances.rb +47 -0
- data/test/functional/lexer/properties.rb +104 -0
- data/test/functional/lexer/syn-pred.rb +32 -0
- data/test/functional/lexer/xml.rb +206 -0
- data/test/functional/main/main-scripts.rb +245 -0
- data/test/functional/parser/actions.rb +224 -0
- data/test/functional/parser/backtracking.rb +244 -0
- data/test/functional/parser/basic.rb +282 -0
- data/test/functional/parser/calc.rb +98 -0
- data/test/functional/parser/ll-star.rb +143 -0
- data/test/functional/parser/nuances.rb +165 -0
- data/test/functional/parser/predicates.rb +103 -0
- data/test/functional/parser/properties.rb +242 -0
- data/test/functional/parser/rule-methods.rb +132 -0
- data/test/functional/parser/scopes.rb +274 -0
- data/test/functional/token-rewrite/basic.rb +318 -0
- data/test/functional/token-rewrite/via-parser.rb +100 -0
- data/test/functional/tree-parser/basic.rb +750 -0
- data/test/unit/sample-input/file-stream-1 +2 -0
- data/test/unit/sample-input/teststreams.input2 +2 -0
- data/test/unit/test-dfa.rb +52 -0
- data/test/unit/test-exceptions.rb +44 -0
- data/test/unit/test-recognizers.rb +55 -0
- data/test/unit/test-scheme.rb +62 -0
- data/test/unit/test-streams.rb +459 -0
- data/test/unit/test-tree-wizard.rb +535 -0
- data/test/unit/test-trees.rb +854 -0
- metadata +205 -0
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
require 'antlr3'
|
5
|
+
require 'test/unit'
|
6
|
+
require 'spec'
|
7
|
+
|
8
|
+
class DFASubclass < ANTLR3::DFA
|
9
|
+
EOT = [1, 2].freeze
|
10
|
+
EOF = [3, 4].freeze
|
11
|
+
MAX = [5, 6].freeze
|
12
|
+
MIN = [7, 8].freeze
|
13
|
+
ACCEPT = [9, 10, 11].freeze
|
14
|
+
SPECIAL = [12].freeze
|
15
|
+
TRANSITION = [
|
16
|
+
[13, 14, 15, 16].freeze,
|
17
|
+
[].freeze
|
18
|
+
].freeze
|
19
|
+
end
|
20
|
+
|
21
|
+
class TestDFA < Test::Unit::TestCase
|
22
|
+
def test_init
|
23
|
+
dfa = DFASubclass.new(nil, 1)
|
24
|
+
dfa.eot.should == DFASubclass::EOT
|
25
|
+
dfa.eof.should == DFASubclass::EOF
|
26
|
+
dfa.max.should == DFASubclass::MAX
|
27
|
+
dfa.min.should == DFASubclass::MIN
|
28
|
+
dfa.accept.should == DFASubclass::ACCEPT
|
29
|
+
dfa.special.should == DFASubclass::SPECIAL
|
30
|
+
dfa.transition.should == DFASubclass::TRANSITION
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_unpack
|
34
|
+
packed = [
|
35
|
+
1, 3, 1, 4, 2, -1, 1, 5, 18, -1, 1, 2,
|
36
|
+
25, -1, 1, 6, 6, -1, 26, 6, 4, -1, 1, 6,
|
37
|
+
1, -1, 26, 6
|
38
|
+
]
|
39
|
+
unpacked = [
|
40
|
+
3, 4, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
41
|
+
-1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
42
|
+
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
|
43
|
+
6, -1, -1, -1, -1, -1, -1, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
44
|
+
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, -1, -1, -1, -1, 6, -1,
|
45
|
+
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
|
46
|
+
6, 6, 6, 6, 6
|
47
|
+
]
|
48
|
+
|
49
|
+
ANTLR3::DFA.unpack(*packed).should == unpacked
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
require 'antlr3'
|
5
|
+
require 'test/unit'
|
6
|
+
require 'spec'
|
7
|
+
|
8
|
+
include ANTLR3::Error
|
9
|
+
#
|
10
|
+
#class TestRecognitionError < Test::Unit::TestCase
|
11
|
+
# def test_init_none
|
12
|
+
# RecognitionError.new()
|
13
|
+
# end
|
14
|
+
#end
|
15
|
+
#
|
16
|
+
#class TestEarlyExit < Test::Unit::TestCase
|
17
|
+
# def test_init_none
|
18
|
+
# EarlyExit.new
|
19
|
+
# end
|
20
|
+
#end
|
21
|
+
#
|
22
|
+
#class TestMismatchedNotSet < Test::Unit::TestCase
|
23
|
+
# def test_init_none
|
24
|
+
# MismatchedNotSet.new
|
25
|
+
# end
|
26
|
+
#end
|
27
|
+
#
|
28
|
+
#class TestMismatchedRange < Test::Unit::TestCase
|
29
|
+
# def test_init_none
|
30
|
+
# MismatchedSet.new
|
31
|
+
# end
|
32
|
+
#end
|
33
|
+
#
|
34
|
+
#class TestMismatchedToken < Test::Unit::TestCase
|
35
|
+
# def test_init_none
|
36
|
+
# MismatchedToken.new
|
37
|
+
# end
|
38
|
+
#end
|
39
|
+
#
|
40
|
+
#class TestNoViableAlternative < Test::Unit::TestCase
|
41
|
+
# def test_init_none
|
42
|
+
# NoViableAlternative.new
|
43
|
+
# end
|
44
|
+
#end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
require 'antlr3'
|
5
|
+
require 'test/unit'
|
6
|
+
require 'spec'
|
7
|
+
|
8
|
+
include ANTLR3
|
9
|
+
|
10
|
+
class TestTokenSource < Test::Unit::TestCase
|
11
|
+
TrivialToken = Struct.new(:type) do
|
12
|
+
include Token
|
13
|
+
end
|
14
|
+
class TestSource
|
15
|
+
include TokenSource
|
16
|
+
def initialize
|
17
|
+
@tokens = (1..4).map { |i| TrivialToken[i] }
|
18
|
+
@tokens << TrivialToken[EOF]
|
19
|
+
end
|
20
|
+
|
21
|
+
def next_token
|
22
|
+
@tokens.shift
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_iterator_interface
|
27
|
+
src = TestSource.new
|
28
|
+
tokens = []
|
29
|
+
src.each do |token|
|
30
|
+
tokens << token.type
|
31
|
+
end
|
32
|
+
tokens.should == [1,2,3,4]
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
class TestLexer < Test::Unit::TestCase
|
38
|
+
class TLexer < Lexer
|
39
|
+
@antlr_version = ANTLR3::ANTLR_VERSION.dup
|
40
|
+
end
|
41
|
+
def test_init
|
42
|
+
stream = StringStream.new('foo')
|
43
|
+
TLexer.new(stream)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
__END__
|
48
|
+
testrecognizers.py | LN | STATUS
|
49
|
+
---------------------------------------------+----+--------------
|
50
|
+
class TestBaseRecognizer(unittest.TestCase) | 07 | [x]
|
51
|
+
def testGetRuleInvocationStack(self) | 10 | [x]
|
52
|
+
class TestTokenSource(unittest.TestCase) | 20 | [x]
|
53
|
+
def testIteratorInterface(self) | 24 | [x]
|
54
|
+
class TestLexer(unittest.TestCase) | 54 | [x]
|
55
|
+
def testInit(self) | 56 | [x]
|
@@ -0,0 +1,62 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
require 'antlr3'
|
4
|
+
require 'test/unit'
|
5
|
+
require 'spec'
|
6
|
+
|
7
|
+
include ANTLR3
|
8
|
+
|
9
|
+
describe TokenScheme do
|
10
|
+
before do
|
11
|
+
@ts = TokenScheme.new do
|
12
|
+
define_tokens(:A => 4, :B => 5, :T__6 => 6)
|
13
|
+
register_names('A', 'B', "'+'")
|
14
|
+
end
|
15
|
+
@a_class = Class.new do
|
16
|
+
class << self
|
17
|
+
attr_accessor :token_scheme
|
18
|
+
end
|
19
|
+
end
|
20
|
+
@a_class.send(:include, @ts)
|
21
|
+
|
22
|
+
@an_instance = @a_class.new
|
23
|
+
end
|
24
|
+
|
25
|
+
example "token schemes define tokens as constants" do
|
26
|
+
@ts::A.should == 4
|
27
|
+
@ts::B.should == 5
|
28
|
+
@ts::T__6.should == 6
|
29
|
+
@ts::EOF.should == -1
|
30
|
+
end
|
31
|
+
|
32
|
+
example "token schemes track human-friendly token names" do
|
33
|
+
@ts::TOKEN_NAMES.should == {
|
34
|
+
0 => "<invalid>", -1 => "<EOF>", 1 => "<EOR>",
|
35
|
+
2 => "<DOWN>", 3 => "<UP>", 4 => "A",
|
36
|
+
5 => "B", 6 => "'+'"
|
37
|
+
}
|
38
|
+
@ts.token_name(5).should == 'B'
|
39
|
+
@ts.token_name(6).should == "'+'"
|
40
|
+
@ts.token_name(-1).should == '<EOF>'
|
41
|
+
@ts.token_name(7).should == '<UNKNOWN: 7>'
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
example 'class-level results of including a token scheme' do
|
46
|
+
#@a_class.token_scheme.should == @ts
|
47
|
+
|
48
|
+
@a_class::A.should == 4
|
49
|
+
@a_class::B.should == 5
|
50
|
+
@a_class::T__6.should == 6
|
51
|
+
@a_class::EOF.should == -1
|
52
|
+
|
53
|
+
@a_class.send(:token_names).should == {
|
54
|
+
0 => "<invalid>", -1 => "<EOF>", 1 => "<EOR>",
|
55
|
+
2 => "<DOWN>", 3 => "<UP>", 4 => "A",
|
56
|
+
5 => "B", 6 => "'+'"
|
57
|
+
}
|
58
|
+
end
|
59
|
+
|
60
|
+
example 'instance-level results of including a token scheme' do
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,459 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
require 'antlr3'
|
5
|
+
require 'test/unit'
|
6
|
+
require 'spec'
|
7
|
+
|
8
|
+
|
9
|
+
include ANTLR3
|
10
|
+
|
11
|
+
class TestStringStream < Test::Unit::TestCase
|
12
|
+
def setup
|
13
|
+
@stream = StringStream.new("oh\nhey!\n")
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_size
|
17
|
+
@stream.size.should == 8
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_index
|
21
|
+
@stream.index.should == 0
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_consume
|
25
|
+
@stream.consume # o
|
26
|
+
@stream.index.should == 1
|
27
|
+
@stream.column.should == 1
|
28
|
+
@stream.line.should == 1
|
29
|
+
|
30
|
+
@stream.consume # h
|
31
|
+
@stream.index.should == 2
|
32
|
+
@stream.column.should == 2
|
33
|
+
@stream.line.should == 1
|
34
|
+
|
35
|
+
@stream.consume # \n
|
36
|
+
@stream.index.should == 3
|
37
|
+
@stream.column.should == 0
|
38
|
+
@stream.line.should == 2
|
39
|
+
|
40
|
+
@stream.consume # h
|
41
|
+
@stream.index.should == 4
|
42
|
+
@stream.column.should == 1
|
43
|
+
@stream.line.should == 2
|
44
|
+
|
45
|
+
@stream.consume # e
|
46
|
+
@stream.index.should == 5
|
47
|
+
@stream.column.should == 2
|
48
|
+
@stream.line.should == 2
|
49
|
+
|
50
|
+
@stream.consume # y
|
51
|
+
@stream.index.should == 6
|
52
|
+
@stream.column.should == 3
|
53
|
+
@stream.line.should == 2
|
54
|
+
|
55
|
+
@stream.consume # !
|
56
|
+
@stream.index.should == 7
|
57
|
+
@stream.column.should == 4
|
58
|
+
@stream.line.should == 2
|
59
|
+
|
60
|
+
@stream.consume # \n
|
61
|
+
@stream.index.should == 8
|
62
|
+
@stream.column.should == 0
|
63
|
+
@stream.line.should == 3
|
64
|
+
|
65
|
+
@stream.consume # EOF
|
66
|
+
@stream.index.should == 8
|
67
|
+
@stream.column.should == 0
|
68
|
+
@stream.line.should == 3
|
69
|
+
|
70
|
+
@stream.consume # EOF
|
71
|
+
@stream.index.should == 8
|
72
|
+
@stream.column.should == 0
|
73
|
+
@stream.line.should == 3
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_reset
|
77
|
+
2.times { @stream.consume }
|
78
|
+
@stream.reset
|
79
|
+
@stream.index.should == 0
|
80
|
+
@stream.line.should == 1
|
81
|
+
@stream.column.should == 0
|
82
|
+
@stream.peek(1).should == ?o
|
83
|
+
end
|
84
|
+
|
85
|
+
def test_look
|
86
|
+
@stream.look(1).should == 'o'
|
87
|
+
@stream.look(2).should == 'h'
|
88
|
+
@stream.look(3).should == "\n"
|
89
|
+
@stream.peek(1).should == ?o
|
90
|
+
@stream.peek(2).should == ?h
|
91
|
+
@stream.peek(3).should == ?\n
|
92
|
+
|
93
|
+
6.times { @stream.consume }
|
94
|
+
@stream.look(1).should == '!'
|
95
|
+
@stream.look(2).should == "\n"
|
96
|
+
@stream.look(3).should be_nil
|
97
|
+
@stream.peek(1).should == ?!
|
98
|
+
@stream.peek(2).should == ?\n
|
99
|
+
@stream.peek(3).should == EOF
|
100
|
+
end
|
101
|
+
|
102
|
+
def test_substring
|
103
|
+
@stream.substring(0,0).should == 'o'
|
104
|
+
@stream.substring(0,1).should == 'oh'
|
105
|
+
@stream.substring(0,8).should == "oh\nhey!\n"
|
106
|
+
@stream.substring(3,6).should == "hey!"
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_seek_forward
|
110
|
+
@stream.seek(3)
|
111
|
+
@stream.index.should == 3
|
112
|
+
@stream.line.should == 2
|
113
|
+
@stream.column.should == 0
|
114
|
+
@stream.peek(1).should == ?h
|
115
|
+
end
|
116
|
+
|
117
|
+
def test_mark
|
118
|
+
@stream.seek(4)
|
119
|
+
marker = @stream.mark
|
120
|
+
marker.should == 1
|
121
|
+
|
122
|
+
2.times { @stream.consume }
|
123
|
+
marker = @stream.mark
|
124
|
+
|
125
|
+
marker.should == 2
|
126
|
+
end
|
127
|
+
|
128
|
+
def test_release_last
|
129
|
+
@stream.seek(4)
|
130
|
+
marker1 = @stream.mark
|
131
|
+
|
132
|
+
2.times { @stream.consume }
|
133
|
+
marker2 = @stream.mark
|
134
|
+
|
135
|
+
@stream.release
|
136
|
+
@stream.mark_depth.should == 2
|
137
|
+
@stream.release
|
138
|
+
@stream.mark_depth.should == 1
|
139
|
+
end
|
140
|
+
|
141
|
+
def test_release_nested
|
142
|
+
@stream.seek(4)
|
143
|
+
marker1 = @stream.mark()
|
144
|
+
|
145
|
+
@stream.consume()
|
146
|
+
marker2 = @stream.mark()
|
147
|
+
|
148
|
+
@stream.consume()
|
149
|
+
marker3 = @stream.mark()
|
150
|
+
|
151
|
+
@stream.release(marker2)
|
152
|
+
@stream.mark_depth.should == 2
|
153
|
+
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_rewind_last
|
157
|
+
@stream.seek(4)
|
158
|
+
|
159
|
+
marker = @stream.mark
|
160
|
+
@stream.consume
|
161
|
+
@stream.consume
|
162
|
+
|
163
|
+
@stream.rewind
|
164
|
+
@stream.mark_depth.should == 1
|
165
|
+
@stream.index.should == 4
|
166
|
+
@stream.line.should == 2
|
167
|
+
@stream.column.should == 1
|
168
|
+
@stream.peek(1).should == ?e
|
169
|
+
|
170
|
+
end
|
171
|
+
|
172
|
+
def test_through
|
173
|
+
@stream.through( 2 ).should == 'oh'
|
174
|
+
@stream.through( -2 ).should == ''
|
175
|
+
@stream.seek( 5 )
|
176
|
+
@stream.through( 0 ).should == ''
|
177
|
+
@stream.through( 1 ).should == 'y'
|
178
|
+
@stream.through( -2 ).should == 'he'
|
179
|
+
@stream.through( 5 ).should == "y!\n"
|
180
|
+
end
|
181
|
+
|
182
|
+
def test_rewind_nested
|
183
|
+
@stream.seek(4)
|
184
|
+
marker1 = @stream.mark()
|
185
|
+
|
186
|
+
@stream.consume
|
187
|
+
marker2 = @stream.mark
|
188
|
+
|
189
|
+
@stream.consume
|
190
|
+
marker3 = @stream.mark
|
191
|
+
|
192
|
+
@stream.rewind(marker2)
|
193
|
+
@stream.mark_depth.should == 2
|
194
|
+
@stream.index().should == 5
|
195
|
+
@stream.line.should == 2
|
196
|
+
@stream.column.should == 2
|
197
|
+
@stream.peek(1).should == ?y
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
class TestFileStream < Test::Unit::TestCase
|
202
|
+
|
203
|
+
|
204
|
+
def test_no_encoding
|
205
|
+
path = File.join(File.dirname(__FILE__), 'sample-input/file-stream-1')
|
206
|
+
@stream = FileStream.new(path)
|
207
|
+
|
208
|
+
@stream.seek(4)
|
209
|
+
marker1 = @stream.mark()
|
210
|
+
|
211
|
+
@stream.consume()
|
212
|
+
marker2 = @stream.mark()
|
213
|
+
|
214
|
+
@stream.consume()
|
215
|
+
marker3 = @stream.mark()
|
216
|
+
|
217
|
+
@stream.rewind(marker2)
|
218
|
+
@stream.index().should == 5
|
219
|
+
@stream.line.should == 2
|
220
|
+
@stream.column.should == 1
|
221
|
+
@stream.mark_depth.should == 2
|
222
|
+
@stream.look(1).should == 'a'
|
223
|
+
@stream.peek(1).should == ?a
|
224
|
+
end
|
225
|
+
|
226
|
+
def test_encoded
|
227
|
+
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
class TestInputStream < Test::Unit::TestCase
|
232
|
+
def test_no_encoding
|
233
|
+
|
234
|
+
end
|
235
|
+
|
236
|
+
def test_encoded
|
237
|
+
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
class TestCommonTokenStream < Test::Unit::TestCase
|
242
|
+
class MockSource
|
243
|
+
include ANTLR3::TokenSource
|
244
|
+
attr_accessor :tokens
|
245
|
+
def initialize
|
246
|
+
@tokens = []
|
247
|
+
end
|
248
|
+
def next_token
|
249
|
+
@tokens.shift
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
# vvvvvvvv tests vvvvvvvvv
|
254
|
+
def test_init
|
255
|
+
@source = MockSource.new
|
256
|
+
@stream = CommonTokenStream.new(@source)
|
257
|
+
@stream.position.should == 0
|
258
|
+
end
|
259
|
+
|
260
|
+
def test_rebuild
|
261
|
+
@source1 = MockSource.new
|
262
|
+
@source2 = MockSource.new
|
263
|
+
@source2.tokens << new_token(10, :channel => ANTLR3::HIDDEN) << new_token(11)
|
264
|
+
@stream = CommonTokenStream.new(@source1)
|
265
|
+
|
266
|
+
@stream.position.should == 0
|
267
|
+
@stream.tokens.should be_empty
|
268
|
+
|
269
|
+
@stream.rebuild(@source2)
|
270
|
+
@stream.token_source.should == @source2
|
271
|
+
@stream.position.should == 1
|
272
|
+
@stream.tokens.should have(2).things
|
273
|
+
end
|
274
|
+
|
275
|
+
def test_look_empty_source
|
276
|
+
@source = MockSource.new
|
277
|
+
@stream = CommonTokenStream.new(@source)
|
278
|
+
@stream.look.should == ANTLR3::EOF_TOKEN
|
279
|
+
end
|
280
|
+
|
281
|
+
def test_look1
|
282
|
+
@source = MockSource.new
|
283
|
+
@source.tokens << new_token(12)
|
284
|
+
@stream = CommonTokenStream.new(@source)
|
285
|
+
@stream.look(1).type.should == 12
|
286
|
+
end
|
287
|
+
|
288
|
+
def test_look1_with_hidden
|
289
|
+
# FIX
|
290
|
+
@source = MockSource.new
|
291
|
+
@source.tokens << new_token(12, :channel => ANTLR3::HIDDEN_CHANNEL) <<
|
292
|
+
new_token(13)
|
293
|
+
@stream = CommonTokenStream.new(@source)
|
294
|
+
@stream.look(1).type.should == 13
|
295
|
+
end
|
296
|
+
|
297
|
+
def test_look2_beyond_end
|
298
|
+
@source = MockSource.new
|
299
|
+
@source.tokens << new_token(12) <<
|
300
|
+
new_token(13, :channel => ANTLR3::HIDDEN_CHANNEL)
|
301
|
+
|
302
|
+
@stream = CommonTokenStream.new(@source)
|
303
|
+
@stream.look(2).type.should == EOF
|
304
|
+
end
|
305
|
+
|
306
|
+
def test_look_negative
|
307
|
+
@source = MockSource.new
|
308
|
+
@source.tokens << new_token(12) << new_token(13)
|
309
|
+
@stream = CommonTokenStream.new(@source)
|
310
|
+
@stream.consume
|
311
|
+
|
312
|
+
@stream.look(-1).type.should == 12
|
313
|
+
end
|
314
|
+
|
315
|
+
def test_lb1
|
316
|
+
@source = MockSource.new
|
317
|
+
@source.tokens << new_token(12) << new_token(13)
|
318
|
+
@stream = CommonTokenStream.new(@source)
|
319
|
+
|
320
|
+
@stream.consume
|
321
|
+
@stream.look(-1).type.should == 12
|
322
|
+
end
|
323
|
+
|
324
|
+
def test_look_zero
|
325
|
+
# FIX
|
326
|
+
@source = MockSource.new
|
327
|
+
@source.tokens << new_token(12) << new_token(13)
|
328
|
+
@stream = CommonTokenStream.new(@source)
|
329
|
+
@stream.look(0).should be_nil
|
330
|
+
end
|
331
|
+
|
332
|
+
def test_lb_beyond_begin
|
333
|
+
@source = MockSource.new
|
334
|
+
@source.tokens << new_token(10) <<
|
335
|
+
new_token(11, :channel => HIDDEN_CHANNEL) <<
|
336
|
+
new_token(12, :channel => HIDDEN_CHANNEL) <<
|
337
|
+
new_token(13)
|
338
|
+
@stream = CommonTokenStream.new(@source)
|
339
|
+
|
340
|
+
@stream.look(-1).should be_nil
|
341
|
+
2.times { @stream.consume }
|
342
|
+
@stream.look(-3).should be_nil
|
343
|
+
end
|
344
|
+
|
345
|
+
def test_fill_buffer
|
346
|
+
@source = MockSource.new
|
347
|
+
@source.tokens << new_token(12) << new_token(13) << new_token(14) << new_token(EOF)
|
348
|
+
@stream = CommonTokenStream.new(@source)
|
349
|
+
|
350
|
+
@stream.instance_variable_get(:@tokens).length.should == 3
|
351
|
+
@stream.tokens[0].type.should == 12
|
352
|
+
@stream.tokens[1].type.should == 13
|
353
|
+
@stream.tokens[2].type.should == 14
|
354
|
+
end
|
355
|
+
|
356
|
+
def test_consume
|
357
|
+
@source = MockSource.new
|
358
|
+
@source.tokens << new_token(12) << new_token(13) << new_token(EOF)
|
359
|
+
@stream = CommonTokenStream.new(@source)
|
360
|
+
@stream.peek.should == 12
|
361
|
+
@stream.consume
|
362
|
+
@stream.peek.should == 13
|
363
|
+
@stream.consume
|
364
|
+
@stream.peek.should == EOF
|
365
|
+
@stream.consume
|
366
|
+
@stream.peek.should == EOF
|
367
|
+
end
|
368
|
+
|
369
|
+
def test_seek
|
370
|
+
@source = MockSource.new
|
371
|
+
@source.tokens << new_token(12) << new_token(13) << new_token(EOF)
|
372
|
+
@stream = CommonTokenStream.new(@source)
|
373
|
+
|
374
|
+
@stream.peek(1).should == 12
|
375
|
+
@stream.seek(2).peek.should == EOF
|
376
|
+
@stream.seek(0).peek.should == 12
|
377
|
+
@stream.seek(-3).position.should == 0
|
378
|
+
@stream.seek(10).position.should == 2
|
379
|
+
end
|
380
|
+
|
381
|
+
def test_mark_rewind
|
382
|
+
@source = MockSource.new
|
383
|
+
@source.tokens << new_token(12) << new_token(13) << new_token(EOF)
|
384
|
+
@stream = CommonTokenStream.new(@source)
|
385
|
+
@stream.consume
|
386
|
+
marker = @stream.mark
|
387
|
+
@stream.consume
|
388
|
+
@stream.rewind(marker)
|
389
|
+
@stream.peek(1).should == 13
|
390
|
+
end
|
391
|
+
|
392
|
+
def test_to_string
|
393
|
+
@source = MockSource.new
|
394
|
+
@source.tokens << new_token(12, 'foo') <<
|
395
|
+
new_token(13, 'bar') << new_token(14, 'gnurz') <<
|
396
|
+
new_token(15, 'blarz')
|
397
|
+
@stream = CommonTokenStream.new(@source)
|
398
|
+
@stream.to_s.should == "foobargnurzblarz"
|
399
|
+
@stream.to_s(1,2).should == 'bargnurz'
|
400
|
+
@stream.to_s(@stream[1], @stream[-2]).should == 'bargnurz'
|
401
|
+
end
|
402
|
+
|
403
|
+
def new_token(type, opts = {})
|
404
|
+
fields = {}
|
405
|
+
case type
|
406
|
+
when Hash then fields.update(type)
|
407
|
+
else
|
408
|
+
fields[:type] = type
|
409
|
+
end
|
410
|
+
case opts
|
411
|
+
when Hash then fields.update(opts)
|
412
|
+
when String then fields[:text] = opts
|
413
|
+
end
|
414
|
+
CommonToken.create(fields)
|
415
|
+
end
|
416
|
+
|
417
|
+
end
|
418
|
+
|
419
|
+
|
420
|
+
__END__
|
421
|
+
teststreams.py | LN | STATUS
|
422
|
+
----------------------------------------------+-----+--------------
|
423
|
+
class TestStringStream(unittest.TestCase) | 009 | [x]
|
424
|
+
def testSize(self) | 012 | [x]
|
425
|
+
def testIndex(self) | 020 | [x]
|
426
|
+
def testConsume(self) | 028 | [x]
|
427
|
+
def testReset(self) | 079 | [x]
|
428
|
+
def testLA(self) | 094 | [x]
|
429
|
+
def testSubstring(self) | 111 | [x]
|
430
|
+
def testSeekForward(self) | 122 | [x]
|
431
|
+
def testMark(self) | 150 | [x]
|
432
|
+
def testReleaseLast(self) | 167 | [x]
|
433
|
+
def testReleaseNested(self) | 186 | [x]
|
434
|
+
def testRewindLast(self) | 204 | [x]
|
435
|
+
def testRewindNested(self) | 223 | [x]
|
436
|
+
class TestFileStream(unittest.TestCase) | 245 | [o]
|
437
|
+
def testNoEncoding(self) | 249 | [x]
|
438
|
+
def testEncoded(self) | 272 | [ ]
|
439
|
+
class TestInputStream(unittest.TestCase) | 296 | [ ]
|
440
|
+
def testNoEncoding(self) | 299 | [ ]
|
441
|
+
def testEncoded(self) | 322 | [ ]
|
442
|
+
class TestCommonTokenStream(unittest.TestCase)| 345 | [ ]
|
443
|
+
def setUp(self) | 348 | [x]
|
444
|
+
def testInit(self) | 369 | [x]
|
445
|
+
def testSetTokenSource(self) | 376 | [x]
|
446
|
+
def testLTEmptySource(self) | 385 | [x]
|
447
|
+
def testLT1(self) | 394 | [x]
|
448
|
+
def testLT1WithHidden(self) | 407 | [x]
|
449
|
+
def testLT2BeyondEnd(self) | 424 | [x]
|
450
|
+
def testLTNegative(self) | 442 | [x]
|
451
|
+
def testLB1(self) | 461 | [x]
|
452
|
+
def testLTZero(self) | 479 | [x]
|
453
|
+
def testLBBeyondBegin(self) | 496 | [x]
|
454
|
+
def testFillBuffer(self) | 523 | [x]
|
455
|
+
def testConsume(self) | 551 | [x]
|
456
|
+
def testSeek(self) | 579 | [x]
|
457
|
+
def testMarkRewind(self) | 604 | [x]
|
458
|
+
def testToString(self) | 631 | [x]
|
459
|
+
|