rubylexer 0.7.7 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -0
- data/History.txt +64 -0
- data/Makefile +2 -2
- data/README.txt +13 -9
- data/bin/rubylexer +113 -0
- data/lib/assert.rb +1 -1
- data/lib/rubylexer.rb +856 -305
- data/lib/rubylexer/charhandler.rb +1 -1
- data/lib/rubylexer/charset.rb +15 -7
- data/lib/rubylexer/context.rb +10 -2
- data/lib/rubylexer/lextable.rb +1 -0
- data/lib/rubylexer/rubycode.rb +1 -1
- data/lib/rubylexer/rulexer.rb +106 -32
- data/lib/rubylexer/symboltable.rb +1 -1
- data/lib/rubylexer/test/oneliners.rb +15 -5
- data/lib/rubylexer/test/oneliners_1.9.rb +116 -92
- data/lib/rubylexer/test/stanzas.rb +49 -27
- data/lib/rubylexer/test/testcases.rb +2 -2
- data/lib/rubylexer/token.rb +153 -23
- data/lib/rubylexer/tokenprinter.rb +9 -6
- data/lib/rubylexer/version.rb +1 -1
- data/rubylexer.gemspec +12 -8
- data/test/bad/ruby_lexer.rb +7 -0
- data/test/code/deletewarns.rb +1 -1
- data/test/code/dumptokens.rb +1 -81
- data/test/code/heredoc_blast_test.rb +112 -0
- data/test/code/locatetest.rb +1 -1
- data/test/code/regression.rb +23 -23
- data/test/code/rubylexervsruby.rb +59 -12
- data/test/code/tokentest.rb +62 -52
- data/test/data/23.rb +0 -1
- data/test/data/g.rb +0 -1
- data/test/data/heremonsters.rb +1 -1
- data/test/data/heremonsters_dos.rb +1 -1
- data/test/data/pre.rb +0 -1
- data/test/data/pre.unix.rb +0 -1
- data/test/data/putstext.rb +4 -0
- data/test/data/regtest.rb +0 -1
- data/test/data/stuffydog.rb +5 -0
- data/test/data/stuffydog2.rb +5 -0
- data/test/data/wsdlDriver.rb +0 -1
- data/test/test.sh +1 -1
- data/test/test_all.rb +3 -0
- data/test/test_bad_rubylexer.rb +16 -0
- data/test/test_rubylexer_bad.rb +12 -0
- data/testing.txt +40 -20
- metadata +51 -38
|
@@ -1,5 +1,47 @@
|
|
|
1
|
+
#encoding: utf-8
|
|
2
|
+
|
|
1
3
|
module Ruby1_9OneLiners
|
|
2
4
|
EXPECT_NO_METHODS=[ #no errors either
|
|
5
|
+
'__ENCODING__',
|
|
6
|
+
'__ENCODING__ +"foo"',
|
|
7
|
+
'module __ENCODING__::A; end',
|
|
8
|
+
'/(?<foo>bar)/ =~ "baz" && foo +1',
|
|
9
|
+
"/(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
10
|
+
'/\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
11
|
+
"/\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
12
|
+
'/\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
13
|
+
"/\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
14
|
+
'/\\\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
15
|
+
"/\\\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
16
|
+
'/\\C-\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
17
|
+
"/\\C-\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
18
|
+
'/\\\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
19
|
+
"/\\\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
20
|
+
'/\\(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
21
|
+
"/\\(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
22
|
+
'/\\[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
23
|
+
"/\\[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
24
|
+
|
|
25
|
+
'/z(?<foo>bar)/ =~ "baz" && foo +1',
|
|
26
|
+
"/z(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
27
|
+
'/z\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
28
|
+
"/z\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
29
|
+
'/z\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
30
|
+
"/z\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
31
|
+
'/z\\\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
32
|
+
"/z\\\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
33
|
+
'/z\\C-\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
34
|
+
"/z\\C-\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
35
|
+
'/z\\\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
36
|
+
"/z\\\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
37
|
+
'/z\\(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
38
|
+
"/z\\(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
39
|
+
'/z\\[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
40
|
+
"/z\\[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
41
|
+
"?\\uaeee",
|
|
42
|
+
"#encoding: utf-8\n ?♥", #utf8 heart char
|
|
43
|
+
'$f.($x,$y)',
|
|
44
|
+
'$f::($x,$y)',
|
|
3
45
|
'->a; h do 123 end',
|
|
4
46
|
'->{}',
|
|
5
47
|
'-> {}',
|
|
@@ -14,50 +56,32 @@ module Ruby1_9OneLiners
|
|
|
14
56
|
'->a,b;c{}',
|
|
15
57
|
'->(a,b;){}',
|
|
16
58
|
'->(a,b;c){}',
|
|
17
|
-
'
|
|
18
|
-
'
|
|
19
|
-
'
|
|
20
|
-
'
|
|
21
|
-
'
|
|
22
|
-
'
|
|
23
|
-
|
|
24
|
-
'
|
|
25
|
-
|
|
26
|
-
'
|
|
27
|
-
|
|
28
|
-
'
|
|
29
|
-
|
|
30
|
-
'
|
|
31
|
-
|
|
32
|
-
'
|
|
33
|
-
|
|
34
|
-
'
|
|
35
|
-
|
|
36
|
-
'
|
|
37
|
-
"/\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
|
|
38
|
-
|
|
39
|
-
'/z(?<foo>bar)/ =~ "baz"; foo +1',
|
|
40
|
-
"/z(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
41
|
-
'/z\\\\(?<foo>bar)/ =~ "baz"; foo +1',
|
|
42
|
-
"/z\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
43
|
-
'/z\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
|
|
44
|
-
"/z\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
45
|
-
'/z\\\\c(?<foo>bar)/ =~ "baz"; foo +1',
|
|
46
|
-
"/z\\\\c(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
47
|
-
'/z\\C-\\(?<foo>bar)/ =~ "baz"; foo +1',
|
|
48
|
-
"/z\\C-\\(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
49
|
-
'/z\\\\C-(?<foo>bar)/ =~ "baz"; foo +1',
|
|
50
|
-
"/z\\\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
51
|
-
'/z\\(?#(?<foo>bar))/ =~ "baz"; foo +1',
|
|
52
|
-
"/z\\(?#(?'foo'bar))/ =~ 'baz'; foo +1",
|
|
53
|
-
'/z\\[(?<foo>bar)]/ =~ "baz"; foo +1',
|
|
54
|
-
"/z\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
|
|
59
|
+
'->a,&b{}',
|
|
60
|
+
'->a,*b{}',
|
|
61
|
+
'->a,*b,c{}',
|
|
62
|
+
'->a,*b;c{}',
|
|
63
|
+
'->a,&b;c{}',
|
|
64
|
+
'->(a,*b){}',
|
|
65
|
+
'->(a,&b){}',
|
|
66
|
+
'->(a,*b,c){}',
|
|
67
|
+
'->(a,*b;c){}',
|
|
68
|
+
'->(a,&b;c){}',
|
|
69
|
+
'->&b{}',
|
|
70
|
+
'->*b{}',
|
|
71
|
+
'->*b,c{}',
|
|
72
|
+
'->*b;c{}',
|
|
73
|
+
'->&b;c{}',
|
|
74
|
+
'->(*b){}',
|
|
75
|
+
'->(&b){}',
|
|
76
|
+
'->(*b,c){}',
|
|
77
|
+
'->(*b;c){}',
|
|
78
|
+
'->(&b;c){}',
|
|
55
79
|
]
|
|
56
80
|
|
|
57
81
|
EXPECT_1_METHOD=[
|
|
58
82
|
'def self.foo; 1 end',
|
|
59
|
-
'->{ foo=1 }
|
|
60
|
-
'->do foo=1 end
|
|
83
|
+
'->{ foo=1 } + foo',
|
|
84
|
+
'->do foo=1 end + foo',
|
|
61
85
|
'def __FILE__.foo; 1 end',
|
|
62
86
|
'def __LINE__.foo; 1 end',
|
|
63
87
|
'def a(b,*c,d) 1 end',
|
|
@@ -89,58 +113,58 @@ module Ruby1_9OneLiners
|
|
|
89
113
|
'def __ENCODING__.foo; 1 end',
|
|
90
114
|
'def Z::__ENCODING__; 342 end',
|
|
91
115
|
#'def Z::__ENCODING__.foo; 1 end', #oops, 2 methods here
|
|
92
|
-
'/\\(?<foo>bar)/ =~ "baz"
|
|
93
|
-
"/\\(?'foo'bar)/ =~ 'baz'
|
|
94
|
-
'/\\c(?<foo>bar)/ =~ "baz"
|
|
95
|
-
"/\\c(?'foo'bar)/ =~ 'baz'
|
|
96
|
-
'/\\C-(?<foo>bar)/ =~ "baz"
|
|
97
|
-
"/\\C-(?'foo'bar)/ =~ 'baz'
|
|
98
|
-
'/\\M-(?<foo>bar)/ =~ "baz"
|
|
99
|
-
"/\\M-(?'foo'bar)/ =~ 'baz'
|
|
100
|
-
'/\\\\\\(?<foo>bar)/ =~ "baz"
|
|
101
|
-
"/\\\\\\(?'foo'bar)/ =~ 'baz'
|
|
102
|
-
'/\\\\c\\(?<foo>bar)/ =~ "baz"
|
|
103
|
-
"/\\\\c\\(?'foo'bar)/ =~ 'baz'
|
|
104
|
-
'/(?#(?<foo>bar))/ =~ "baz"
|
|
105
|
-
"/(?#(?'foo'bar))/ =~ 'baz'
|
|
106
|
-
'/[(?<foo>bar)]/ =~ "baz"
|
|
107
|
-
"/[(?'foo'bar)]/ =~ 'baz'
|
|
108
|
-
'/[qwe&&[^ty](?<foo>bar)]/ =~ "baz"
|
|
109
|
-
"/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'
|
|
110
|
-
'/[qwe&&[^(?<foo>bar)]]/ =~ "baz"
|
|
111
|
-
"/[qwe&&[^(?'foo'bar)]]/ =~ 'baz'
|
|
112
|
-
'/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"
|
|
113
|
-
"/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'
|
|
114
|
-
'/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"
|
|
115
|
-
"/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'
|
|
116
|
-
'/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"
|
|
117
|
-
"/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'
|
|
116
|
+
'/\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
117
|
+
"/\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
118
|
+
'/\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
119
|
+
"/\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
120
|
+
'/\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
121
|
+
"/\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
122
|
+
'/\\M-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
123
|
+
"/\\M-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
124
|
+
'/\\\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
125
|
+
"/\\\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
126
|
+
'/\\\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
127
|
+
"/\\\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
128
|
+
'/(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
129
|
+
"/(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
130
|
+
'/[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
131
|
+
"/[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
132
|
+
'/[qwe&&[^ty](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
133
|
+
"/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
134
|
+
'/[qwe&&[^(?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
135
|
+
"/[qwe&&[^(?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
136
|
+
'/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
137
|
+
"/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
138
|
+
'/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
139
|
+
"/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
140
|
+
'/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz" && foo +1',
|
|
141
|
+
"/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz' && foo +1",
|
|
118
142
|
|
|
119
|
-
'/z\\(?<foo>bar)/ =~ "baz"
|
|
120
|
-
"/z\\(?'foo'bar)/ =~ 'baz'
|
|
121
|
-
'/z\\c(?<foo>bar)/ =~ "baz"
|
|
122
|
-
"/z\\c(?'foo'bar)/ =~ 'baz'
|
|
123
|
-
'/z\\C-(?<foo>bar)/ =~ "baz"
|
|
124
|
-
"/z\\C-(?'foo'bar)/ =~ 'baz'
|
|
125
|
-
'/z\\M-(?<foo>bar)/ =~ "baz"
|
|
126
|
-
"/z\\M-(?'foo'bar)/ =~ 'baz'
|
|
127
|
-
'/z\\\\\\(?<foo>bar)/ =~ "baz"
|
|
128
|
-
"/z\\\\\\(?'foo'bar)/ =~ 'baz'
|
|
129
|
-
'/z\\\\c\\(?<foo>bar)/ =~ "baz"
|
|
130
|
-
"/z\\\\c\\(?'foo'bar)/ =~ 'baz'
|
|
131
|
-
'/z(?#(?<foo>bar))/ =~ "baz"
|
|
132
|
-
"/z(?#(?'foo'bar))/ =~ 'baz'
|
|
133
|
-
'/z[(?<foo>bar)]/ =~ "baz"
|
|
134
|
-
"/z[(?'foo'bar)]/ =~ 'baz'
|
|
135
|
-
'/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz"
|
|
136
|
-
"/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'
|
|
137
|
-
'/z[qwe&&[^(?<foo>bar)]]/ =~ "baz"
|
|
138
|
-
"/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz'
|
|
139
|
-
'/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"
|
|
140
|
-
"/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'
|
|
141
|
-
'/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"
|
|
142
|
-
"/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'
|
|
143
|
-
'/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"
|
|
144
|
-
"/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'
|
|
143
|
+
'/z\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
144
|
+
"/z\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
145
|
+
'/z\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
146
|
+
"/z\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
147
|
+
'/z\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
148
|
+
"/z\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
149
|
+
'/z\\M-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
150
|
+
"/z\\M-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
151
|
+
'/z\\\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
152
|
+
"/z\\\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
153
|
+
'/z\\\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
154
|
+
"/z\\\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
155
|
+
'/z(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
156
|
+
"/z(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
157
|
+
'/z[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
158
|
+
"/z[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
159
|
+
'/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
160
|
+
"/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
161
|
+
'/z[qwe&&[^(?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
162
|
+
"/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
163
|
+
'/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
164
|
+
"/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
165
|
+
'/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
166
|
+
"/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
167
|
+
'/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz" && foo +1',
|
|
168
|
+
"/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz' && foo +1",
|
|
145
169
|
]
|
|
146
170
|
end
|
|
@@ -61,9 +61,6 @@ p %(1)
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
=begin
|
|
65
|
-
=end
|
|
66
|
-
|
|
67
64
|
p <<p
|
|
68
65
|
\
|
|
69
66
|
p
|
|
@@ -728,23 +725,10 @@ x {
|
|
|
728
725
|
p(p %(c))
|
|
729
726
|
}
|
|
730
727
|
|
|
731
|
-
=begin not supported yet
|
|
732
728
|
p "#{<<kekerz}#{"foob"
|
|
733
729
|
zimpler
|
|
734
730
|
kekerz
|
|
735
731
|
}"
|
|
736
|
-
=end
|
|
737
|
-
|
|
738
|
-
=begin not supported yet
|
|
739
|
-
aaa=<<whatnot; p "#{'uh,yeah'
|
|
740
|
-
gonna take it down, to the nitty-grit
|
|
741
|
-
gonna tell you mother-fuckers why you ain't shit
|
|
742
|
-
cause suckers like you just make me strong
|
|
743
|
-
you been pumpin' that bullshit all day long
|
|
744
|
-
whatnot
|
|
745
|
-
}"
|
|
746
|
-
p aaa
|
|
747
|
-
=end
|
|
748
732
|
|
|
749
733
|
#test variable creation in string inclusion
|
|
750
734
|
#currently broken because string inclusions
|
|
@@ -761,13 +745,11 @@ p %w[well, whaddaya know].map{|j| <<-END }
|
|
|
761
745
|
}45634543
|
|
762
746
|
END
|
|
763
747
|
|
|
764
|
-
=begin not supported yet
|
|
765
748
|
p "#{<<foobar3}"
|
|
766
749
|
bim
|
|
767
750
|
baz
|
|
768
751
|
bof
|
|
769
752
|
foobar3
|
|
770
|
-
=end
|
|
771
753
|
|
|
772
754
|
x do
|
|
773
755
|
a,b,* = [1,2,3,4,5,6,7,8]
|
|
@@ -947,25 +929,25 @@ p(/
|
|
|
947
929
|
p <<stuff+'foobar'.tr('j-l','d-f')
|
|
948
930
|
"more stuff"
|
|
949
931
|
12345678
|
|
950
|
-
the quick brown fox jumped over the lazy
|
|
932
|
+
the quick brown fox jumped over the lazy dog0
|
|
951
933
|
stuff
|
|
952
934
|
|
|
953
935
|
p <<stuff+'foobar'.tr('j-l','d-f')
|
|
954
936
|
"more stuff"
|
|
955
937
|
12345678
|
|
956
|
-
the quick brown fox jumped over the lazy
|
|
938
|
+
the quick brown fox jumped over the lazy dog1
|
|
957
939
|
stuff
|
|
958
940
|
|
|
959
941
|
p <<stuff+'foobar'.tr('j-l','d-f')\
|
|
960
942
|
+"more stuff"
|
|
961
943
|
12345678
|
|
962
|
-
the quick brown fox jumped over the lazy
|
|
944
|
+
the quick brown fox jumped over the lazy dog2
|
|
963
945
|
stuff
|
|
964
946
|
|
|
965
947
|
p <<stuff+'foobar'\
|
|
966
948
|
+"more stuff"
|
|
967
949
|
12345678
|
|
968
|
-
the quick brown fox jumped over the lazy
|
|
950
|
+
the quick brown fox jumped over the lazy dog3
|
|
969
951
|
stuff
|
|
970
952
|
|
|
971
953
|
|
|
@@ -1100,11 +1082,6 @@ for i in (begin
|
|
|
1100
1082
|
[44,55,66,77,88] end) do p i**Math.sqrt(i) end
|
|
1101
1083
|
}
|
|
1102
1084
|
|
|
1103
|
-
x{
|
|
1104
|
-
for i in if false then foob12345; else
|
|
1105
|
-
[44,55,66,77,88] end do p i**Math.sqrt(i) end
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
1085
|
x{
|
|
1109
1086
|
for i in (
|
|
1110
1087
|
[44,55,66,77,88]) do p i**Math.sqrt(i) end
|
|
@@ -1185,8 +1162,53 @@ sdsdfsdfs
|
|
|
1185
1162
|
A
|
|
1186
1163
|
|
|
1187
1164
|
|
|
1165
|
+
module
|
|
1166
|
+
=begin
|
|
1167
|
+
=end
|
|
1168
|
+
A; end
|
|
1169
|
+
|
|
1170
|
+
module
|
|
1171
|
+
=begin
|
|
1172
|
+
foo
|
|
1173
|
+
=end
|
|
1174
|
+
A; end
|
|
1175
|
+
|
|
1176
|
+
module
|
|
1177
|
+
=begin
|
|
1178
|
+
=end
|
|
1179
|
+
A::
|
|
1180
|
+
=begin
|
|
1181
|
+
=end
|
|
1182
|
+
B; end
|
|
1183
|
+
|
|
1184
|
+
module
|
|
1185
|
+
#=begin
|
|
1186
|
+
#=end
|
|
1187
|
+
A::
|
|
1188
|
+
#=begin
|
|
1189
|
+
#=end
|
|
1190
|
+
B; end
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
module
|
|
1194
|
+
=begin
|
|
1195
|
+
foo
|
|
1196
|
+
=end
|
|
1197
|
+
A::
|
|
1198
|
+
=begin
|
|
1199
|
+
bar
|
|
1200
|
+
=end
|
|
1201
|
+
B; end
|
|
1188
1202
|
|
|
1203
|
+
<<x.
|
|
1204
|
+
1111
|
|
1205
|
+
x
|
|
1206
|
+
delete()
|
|
1189
1207
|
|
|
1208
|
+
<<x::
|
|
1209
|
+
1111
|
|
1210
|
+
x
|
|
1211
|
+
delete()
|
|
1190
1212
|
|
|
1191
1213
|
\
|
|
1192
1214
|
__END__
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
module TestCases
|
|
2
2
|
# fail unless File.exist 'test/data/oneliners.rb' and File.exist 'test/data/stanzas.rb'
|
|
3
3
|
rldir=$:.find{|dir| File.exist? dir+'/rubylexer/test/oneliners.rb' and File.exist? dir+'/rubylexer/test/stanzas.rb' }
|
|
4
|
-
ONELINERS=IO.readlines(rldir+'/rubylexer/test/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s
|
|
4
|
+
ONELINERS=IO.readlines(rldir+'/rubylexer/test/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s]/).reverse
|
|
5
5
|
STANZAS=IO.read(rldir+'/rubylexer/test/stanzas.rb').split("\n\n").grep(/./).reverse
|
|
6
6
|
STANZAS.each{|stanza| stanza<<"\n" }
|
|
7
|
-
ILLEGAL_ONELINERS=IO.readlines(rldir+'/rubylexer/test/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s
|
|
7
|
+
ILLEGAL_ONELINERS=IO.readlines(rldir+'/rubylexer/test/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s]/).reverse
|
|
8
8
|
ILLEGAL_STANZAS=IO.read(rldir+'/rubylexer/test/illegal_stanzas.rb').split("\n\n").grep(/./).reverse
|
|
9
9
|
|
|
10
10
|
datadir=$:.find{|dir| File.exist? dir+'/../test/data/p.rb' }
|
data/lib/rubylexer/token.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
=begin
|
|
1
|
+
=begin
|
|
2
2
|
rubylexer - a ruby lexer written in ruby
|
|
3
|
-
Copyright (C) 2004,2005,2008 Caleb Clausen
|
|
3
|
+
Copyright (C) 2004,2005,2008, 2011 Caleb Clausen
|
|
4
4
|
|
|
5
5
|
This library is free software; you can redistribute it and/or
|
|
6
6
|
modify it under the terms of the GNU Lesser General Public
|
|
@@ -29,6 +29,7 @@ class Token
|
|
|
29
29
|
attr_accessor :offset #file offset of start of this token
|
|
30
30
|
attr_accessor :as #if defined, a KeywordToken which this token stands in for.
|
|
31
31
|
attr_accessor :allow_ooo_offset #hack
|
|
32
|
+
attr_accessor :endline
|
|
32
33
|
|
|
33
34
|
def initialize(ident,offset=nil)
|
|
34
35
|
@ident=ident
|
|
@@ -40,6 +41,29 @@ class Token
|
|
|
40
41
|
def has_no_block?; false end
|
|
41
42
|
|
|
42
43
|
attr_accessor :tag
|
|
44
|
+
|
|
45
|
+
attr_writer :startline
|
|
46
|
+
def startline
|
|
47
|
+
return @startline if defined? @startline
|
|
48
|
+
return endline
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def linecount; 0 end
|
|
52
|
+
|
|
53
|
+
alias orig_inspect inspect
|
|
54
|
+
alias dump inspect
|
|
55
|
+
|
|
56
|
+
#this is merely the normal definition of inspect
|
|
57
|
+
#and is unneeded in ruby 1.8
|
|
58
|
+
#but in 1.9, defining to_s seemingly overrides the built-in Object#inspect
|
|
59
|
+
#and you can't get it back, no matter what.
|
|
60
|
+
#fucking 1.9
|
|
61
|
+
def inspect
|
|
62
|
+
ivars=instance_variables.map{|ivar|
|
|
63
|
+
ivar.to_s+'='+instance_variable_get(ivar).inspect
|
|
64
|
+
}.join(' ')
|
|
65
|
+
%[#<#{self.class}: #{ivars}>]
|
|
66
|
+
end
|
|
43
67
|
end
|
|
44
68
|
|
|
45
69
|
#-------------------------
|
|
@@ -52,6 +76,17 @@ end
|
|
|
52
76
|
|
|
53
77
|
#-------------------------
|
|
54
78
|
class KeywordToken < WToken #also some operators
|
|
79
|
+
def initialize(*args)
|
|
80
|
+
if Hash===args.last
|
|
81
|
+
opts=args.pop
|
|
82
|
+
as=opts.delete :as
|
|
83
|
+
fail unless opts.empty?
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
super(*args)
|
|
87
|
+
self.as=as
|
|
88
|
+
end
|
|
89
|
+
|
|
55
90
|
|
|
56
91
|
#-----------------------------------
|
|
57
92
|
def set_callsite!(x=true)
|
|
@@ -109,8 +144,14 @@ end
|
|
|
109
144
|
|
|
110
145
|
#-------------------------
|
|
111
146
|
class OperatorToken < WToken
|
|
112
|
-
|
|
147
|
+
def initialize(*args)
|
|
148
|
+
@tag=nil
|
|
149
|
+
super
|
|
150
|
+
end
|
|
113
151
|
attr_writer :as
|
|
152
|
+
|
|
153
|
+
def unary= flag; @tag=:unary if flag end
|
|
154
|
+
def unary; @tag==:unary end
|
|
114
155
|
alias prefix? unary
|
|
115
156
|
def infix?; !prefix? end
|
|
116
157
|
|
|
@@ -149,8 +190,20 @@ end
|
|
|
149
190
|
|
|
150
191
|
#-------------------------
|
|
151
192
|
class NumberToken < Token
|
|
152
|
-
def to_s
|
|
153
|
-
|
|
193
|
+
def to_s
|
|
194
|
+
if defined? @char_literal and @char_literal
|
|
195
|
+
chr=@ident.chr
|
|
196
|
+
'?'+case chr
|
|
197
|
+
when " "; '\s'
|
|
198
|
+
when /[!-~]/; chr
|
|
199
|
+
else chr.inspect[1...-1]
|
|
200
|
+
end
|
|
201
|
+
else
|
|
202
|
+
@ident.to_s
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
def negative; /\A-/ === to_s end
|
|
206
|
+
attr_accessor :char_literal
|
|
154
207
|
end
|
|
155
208
|
|
|
156
209
|
#-------------------------
|
|
@@ -207,11 +260,18 @@ end
|
|
|
207
260
|
|
|
208
261
|
#-------------------------
|
|
209
262
|
class NewlineToken < Token
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
263
|
+
def initialize(nlstr="\n",offset=nil)
|
|
264
|
+
super(nlstr,offset)
|
|
265
|
+
#@char=''
|
|
266
|
+
end
|
|
267
|
+
def as; ';' end
|
|
268
|
+
|
|
269
|
+
def linecount; 1 end
|
|
270
|
+
|
|
271
|
+
def startline
|
|
272
|
+
@endline-1
|
|
273
|
+
end
|
|
274
|
+
def startline=bogus; end
|
|
215
275
|
end
|
|
216
276
|
|
|
217
277
|
#-------------------------
|
|
@@ -221,19 +281,31 @@ class StringToken < Token
|
|
|
221
281
|
attr_accessor :modifiers #for regex only
|
|
222
282
|
attr_accessor :elems
|
|
223
283
|
attr_accessor :startline
|
|
224
|
-
attr_accessor :line #line on which the string ENDS
|
|
225
284
|
attr_accessor :bs_handler
|
|
226
285
|
|
|
227
286
|
attr_accessor :open #exact sequence of chars used to start the str
|
|
228
287
|
attr_accessor :close #exact seq of (1) char to stop the str
|
|
229
288
|
|
|
230
289
|
attr_accessor :lvars #names used in named backrefs if this is a regex
|
|
290
|
+
|
|
291
|
+
def linecount; line-startline end
|
|
231
292
|
|
|
293
|
+
def utf8?
|
|
294
|
+
@utf8||=nil
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def utf8!
|
|
298
|
+
@utf8=true
|
|
299
|
+
end
|
|
300
|
+
|
|
232
301
|
def with_line(line)
|
|
233
|
-
@
|
|
302
|
+
@endline=line
|
|
234
303
|
self
|
|
235
304
|
end
|
|
236
305
|
|
|
306
|
+
def line; @endline end
|
|
307
|
+
def line= l; @endline=l end
|
|
308
|
+
|
|
237
309
|
def initialize(type='"',ident='')
|
|
238
310
|
super(ident)
|
|
239
311
|
type=="'" and type='"'
|
|
@@ -241,7 +313,7 @@ class StringToken < Token
|
|
|
241
313
|
assert @char[/^[\[{"`\/]$/] #"
|
|
242
314
|
@elems=[ident.dup] #why .dup?
|
|
243
315
|
@modifiers=nil
|
|
244
|
-
@
|
|
316
|
+
@endline=nil
|
|
245
317
|
end
|
|
246
318
|
|
|
247
319
|
DQUOTE_ESCAPE_TABLE = [
|
|
@@ -291,6 +363,8 @@ class StringToken < Token
|
|
|
291
363
|
result="%r"+result if RubyLexer::WHSPLF[result[1,1]]
|
|
292
364
|
end
|
|
293
365
|
|
|
366
|
+
result<<"\n" if open.empty? and result[-1] != ?\n
|
|
367
|
+
|
|
294
368
|
return result
|
|
295
369
|
end
|
|
296
370
|
|
|
@@ -362,9 +436,9 @@ private
|
|
|
362
436
|
UNESC_DELIMS={}
|
|
363
437
|
|
|
364
438
|
#simpler transform, preserves original exactly
|
|
365
|
-
def simple_transform(strfrag,starter,ender)
|
|
439
|
+
def simple_transform(strfrag,starter,ender) #appears to be unused
|
|
366
440
|
assert('[{/'[@char])
|
|
367
|
-
#strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='['
|
|
441
|
+
#strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='[' #esc #{
|
|
368
442
|
delimchars=Regexp.quote starter+ender
|
|
369
443
|
delimchars+=Regexp.quote("#") unless @char=='[' #escape beginning of string iterpolations
|
|
370
444
|
|
|
@@ -390,7 +464,7 @@ private
|
|
|
390
464
|
return strfrag
|
|
391
465
|
end
|
|
392
466
|
|
|
393
|
-
def transform(strfrag,starter,ender)
|
|
467
|
+
def transform(strfrag,starter,ender) #appears to be unused
|
|
394
468
|
strfrag.gsub!("\\",'\\'*4)
|
|
395
469
|
strfrag.gsub!(/#([{$@])/,'\\#\\1')
|
|
396
470
|
strfrag.gsub!(Regexp.new("[\\"+starter+"\\"+ender+"]"),'\\\\\\&') unless @char=='?'
|
|
@@ -427,8 +501,9 @@ private
|
|
|
427
501
|
end
|
|
428
502
|
|
|
429
503
|
#-------------------------
|
|
430
|
-
class RenderExactlyStringToken < StringToken
|
|
504
|
+
class RenderExactlyStringToken < StringToken
|
|
431
505
|
alias transform simple_transform
|
|
506
|
+
#transform isn't called anymore, so there's no need for this hacky class
|
|
432
507
|
end
|
|
433
508
|
|
|
434
509
|
#-------------------------
|
|
@@ -471,9 +546,13 @@ class HerePlaceholderToken < WToken
|
|
|
471
546
|
|
|
472
547
|
#def with_line(line) @string.line=line; self end
|
|
473
548
|
|
|
474
|
-
def line; @string.line end
|
|
475
|
-
def line=line; @
|
|
549
|
+
def line; @line || @string.line end
|
|
550
|
+
def line=line; @line=line end
|
|
476
551
|
|
|
552
|
+
def startline; @line end
|
|
553
|
+
alias endline startline
|
|
554
|
+
def startline=x; end
|
|
555
|
+
alias endline= startline=
|
|
477
556
|
end
|
|
478
557
|
|
|
479
558
|
#-------------------------
|
|
@@ -484,6 +563,13 @@ end
|
|
|
484
563
|
#-------------------------
|
|
485
564
|
class IgnoreToken < Token
|
|
486
565
|
include StillIgnoreToken
|
|
566
|
+
|
|
567
|
+
def initialize(ident,*stuff)
|
|
568
|
+
@linecount=ident.count "\n"
|
|
569
|
+
super
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
attr :linecount
|
|
487
573
|
end
|
|
488
574
|
|
|
489
575
|
#-------------------------
|
|
@@ -511,6 +597,23 @@ class NoWsToken < ZwToken
|
|
|
511
597
|
end
|
|
512
598
|
end
|
|
513
599
|
|
|
600
|
+
#-------------------------
|
|
601
|
+
class ShebangToken < IgnoreToken
|
|
602
|
+
def initialize(text)
|
|
603
|
+
super text,0
|
|
604
|
+
end
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
#-------------------------
|
|
608
|
+
class EncodingDeclToken < IgnoreToken
|
|
609
|
+
def initialize(text,encoding,offset)
|
|
610
|
+
text||=''
|
|
611
|
+
super text,offset
|
|
612
|
+
@encoding=encoding
|
|
613
|
+
end
|
|
614
|
+
attr :encoding
|
|
615
|
+
end
|
|
616
|
+
|
|
514
617
|
#-------------------------
|
|
515
618
|
class ImplicitParamListStartToken < KeywordToken
|
|
516
619
|
include StillIgnoreToken
|
|
@@ -579,6 +682,19 @@ class EscNlToken < IgnoreToken
|
|
|
579
682
|
@filename=filename
|
|
580
683
|
@linenum=linenum
|
|
581
684
|
end
|
|
685
|
+
|
|
686
|
+
attr_accessor :filename,:linenum
|
|
687
|
+
|
|
688
|
+
def linecount; 1 end
|
|
689
|
+
|
|
690
|
+
def startline
|
|
691
|
+
@linenum-1
|
|
692
|
+
end
|
|
693
|
+
def endline
|
|
694
|
+
@linenum
|
|
695
|
+
end
|
|
696
|
+
def startline= bogus; end
|
|
697
|
+
alias endline= linenum=
|
|
582
698
|
end
|
|
583
699
|
|
|
584
700
|
#-------------------------
|
|
@@ -586,9 +702,10 @@ class EoiToken < Token
|
|
|
586
702
|
attr :file
|
|
587
703
|
alias :pos :offset
|
|
588
704
|
|
|
589
|
-
def initialize(cause,file, offset=nil)
|
|
705
|
+
def initialize(cause,file, offset=nil,line=nil)
|
|
590
706
|
super(cause,offset)
|
|
591
707
|
@file=file
|
|
708
|
+
@endline=line
|
|
592
709
|
end
|
|
593
710
|
end
|
|
594
711
|
|
|
@@ -598,7 +715,7 @@ class HereBodyToken < IgnoreToken
|
|
|
598
715
|
attr_accessor :open,:close
|
|
599
716
|
def initialize(headtok,linecount)
|
|
600
717
|
assert HerePlaceholderToken===headtok
|
|
601
|
-
|
|
718
|
+
@ident,@offset=headtok.string,headtok.string.offset
|
|
602
719
|
@headtok=headtok
|
|
603
720
|
@linecount=linecount
|
|
604
721
|
end
|
|
@@ -606,6 +723,14 @@ class HereBodyToken < IgnoreToken
|
|
|
606
723
|
def line
|
|
607
724
|
@ident.line
|
|
608
725
|
end
|
|
726
|
+
alias endline line
|
|
727
|
+
def endline= line
|
|
728
|
+
@ident.line= line
|
|
729
|
+
end
|
|
730
|
+
|
|
731
|
+
def startline
|
|
732
|
+
line-@linecount+1
|
|
733
|
+
end
|
|
609
734
|
|
|
610
735
|
def to_s
|
|
611
736
|
@ident.to_s
|
|
@@ -629,15 +754,20 @@ class FileAndLineToken < IgnoreToken
|
|
|
629
754
|
#def char; '#' end
|
|
630
755
|
|
|
631
756
|
def to_s()
|
|
632
|
-
[
|
|
757
|
+
%[##@ident:#@line]
|
|
633
758
|
end
|
|
634
759
|
|
|
635
760
|
def file() @ident end
|
|
636
761
|
def subitem() @line end #needed?
|
|
762
|
+
|
|
763
|
+
def endline; @line end
|
|
764
|
+
def startline; @line end
|
|
765
|
+
alias endline= line=
|
|
766
|
+
def startline= bogus; end
|
|
637
767
|
end
|
|
638
768
|
|
|
639
769
|
#-------------------------
|
|
640
|
-
class OutlinedHereBodyToken < HereBodyToken
|
|
770
|
+
class OutlinedHereBodyToken < HereBodyToken #appears to be unused
|
|
641
771
|
def to_s
|
|
642
772
|
assert HerePlaceholderToken===@headtok
|
|
643
773
|
result=@headtok.string
|