rubylexer 0.7.7 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -0
- data/History.txt +64 -0
- data/Makefile +2 -2
- data/README.txt +13 -9
- data/bin/rubylexer +113 -0
- data/lib/assert.rb +1 -1
- data/lib/rubylexer.rb +856 -305
- data/lib/rubylexer/charhandler.rb +1 -1
- data/lib/rubylexer/charset.rb +15 -7
- data/lib/rubylexer/context.rb +10 -2
- data/lib/rubylexer/lextable.rb +1 -0
- data/lib/rubylexer/rubycode.rb +1 -1
- data/lib/rubylexer/rulexer.rb +106 -32
- data/lib/rubylexer/symboltable.rb +1 -1
- data/lib/rubylexer/test/oneliners.rb +15 -5
- data/lib/rubylexer/test/oneliners_1.9.rb +116 -92
- data/lib/rubylexer/test/stanzas.rb +49 -27
- data/lib/rubylexer/test/testcases.rb +2 -2
- data/lib/rubylexer/token.rb +153 -23
- data/lib/rubylexer/tokenprinter.rb +9 -6
- data/lib/rubylexer/version.rb +1 -1
- data/rubylexer.gemspec +12 -8
- data/test/bad/ruby_lexer.rb +7 -0
- data/test/code/deletewarns.rb +1 -1
- data/test/code/dumptokens.rb +1 -81
- data/test/code/heredoc_blast_test.rb +112 -0
- data/test/code/locatetest.rb +1 -1
- data/test/code/regression.rb +23 -23
- data/test/code/rubylexervsruby.rb +59 -12
- data/test/code/tokentest.rb +62 -52
- data/test/data/23.rb +0 -1
- data/test/data/g.rb +0 -1
- data/test/data/heremonsters.rb +1 -1
- data/test/data/heremonsters_dos.rb +1 -1
- data/test/data/pre.rb +0 -1
- data/test/data/pre.unix.rb +0 -1
- data/test/data/putstext.rb +4 -0
- data/test/data/regtest.rb +0 -1
- data/test/data/stuffydog.rb +5 -0
- data/test/data/stuffydog2.rb +5 -0
- data/test/data/wsdlDriver.rb +0 -1
- data/test/test.sh +1 -1
- data/test/test_all.rb +3 -0
- data/test/test_bad_rubylexer.rb +16 -0
- data/test/test_rubylexer_bad.rb +12 -0
- data/testing.txt +40 -20
- metadata +51 -38
|
@@ -1,5 +1,47 @@
|
|
|
1
|
+
#encoding: utf-8
|
|
2
|
+
|
|
1
3
|
module Ruby1_9OneLiners
|
|
2
4
|
EXPECT_NO_METHODS=[ #no errors either
|
|
5
|
+
'__ENCODING__',
|
|
6
|
+
'__ENCODING__ +"foo"',
|
|
7
|
+
'module __ENCODING__::A; end',
|
|
8
|
+
'/(?<foo>bar)/ =~ "baz" && foo +1',
|
|
9
|
+
"/(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
10
|
+
'/\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
11
|
+
"/\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
12
|
+
'/\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
13
|
+
"/\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
14
|
+
'/\\\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
15
|
+
"/\\\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
16
|
+
'/\\C-\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
17
|
+
"/\\C-\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
18
|
+
'/\\\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
19
|
+
"/\\\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
20
|
+
'/\\(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
21
|
+
"/\\(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
22
|
+
'/\\[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
23
|
+
"/\\[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
24
|
+
|
|
25
|
+
'/z(?<foo>bar)/ =~ "baz" && foo +1',
|
|
26
|
+
"/z(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
27
|
+
'/z\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
28
|
+
"/z\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
29
|
+
'/z\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
30
|
+
"/z\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
31
|
+
'/z\\\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
32
|
+
"/z\\\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
33
|
+
'/z\\C-\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
34
|
+
"/z\\C-\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
35
|
+
'/z\\\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
36
|
+
"/z\\\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
37
|
+
'/z\\(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
38
|
+
"/z\\(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
39
|
+
'/z\\[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
40
|
+
"/z\\[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
41
|
+
"?\\uaeee",
|
|
42
|
+
"#encoding: utf-8\n ?♥", #utf8 heart char
|
|
43
|
+
'$f.($x,$y)',
|
|
44
|
+
'$f::($x,$y)',
|
|
3
45
|
'->a; h do 123 end',
|
|
4
46
|
'->{}',
|
|
5
47
|
'-> {}',
|
|
@@ -14,50 +56,32 @@ module Ruby1_9OneLiners
|
|
|
14
56
|
'->a,b;c{}',
|
|
15
57
|
'->(a,b;){}',
|
|
16
58
|
'->(a,b;c){}',
|
|
17
|
-
'
|
|
18
|
-
'
|
|
19
|
-
'
|
|
20
|
-
'
|
|
21
|
-
'
|
|
22
|
-
'
|
|
23
|
-
|
|
24
|
-
'
|
|
25
|
-
|
|
26
|
-
'
|
|
27
|
-
|
|
28
|
-
'
|
|
29
|
-
|
|
30
|
-
'
|
|
31
|
-
|
|
32
|
-
'
|
|
33
|
-
|
|
34
|
-
'
|
|
35
|
-
|
|
36
|
-
'
|
|
37
|
-
"/\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
|
|
38
|
-
|
|
39
|
-
'/z(?<foo>bar)/ =~ "baz"; foo +1',
|
|
40
|
-
"/z(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
41
|
-
'/z\\\\(?<foo>bar)/ =~ "baz"; foo +1',
|
|
42
|
-
"/z\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
43
|
-
'/z\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
|
|
44
|
-
"/z\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
45
|
-
'/z\\\\c(?<foo>bar)/ =~ "baz"; foo +1',
|
|
46
|
-
"/z\\\\c(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
47
|
-
'/z\\C-\\(?<foo>bar)/ =~ "baz"; foo +1',
|
|
48
|
-
"/z\\C-\\(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
49
|
-
'/z\\\\C-(?<foo>bar)/ =~ "baz"; foo +1',
|
|
50
|
-
"/z\\\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
|
|
51
|
-
'/z\\(?#(?<foo>bar))/ =~ "baz"; foo +1',
|
|
52
|
-
"/z\\(?#(?'foo'bar))/ =~ 'baz'; foo +1",
|
|
53
|
-
'/z\\[(?<foo>bar)]/ =~ "baz"; foo +1',
|
|
54
|
-
"/z\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
|
|
59
|
+
'->a,&b{}',
|
|
60
|
+
'->a,*b{}',
|
|
61
|
+
'->a,*b,c{}',
|
|
62
|
+
'->a,*b;c{}',
|
|
63
|
+
'->a,&b;c{}',
|
|
64
|
+
'->(a,*b){}',
|
|
65
|
+
'->(a,&b){}',
|
|
66
|
+
'->(a,*b,c){}',
|
|
67
|
+
'->(a,*b;c){}',
|
|
68
|
+
'->(a,&b;c){}',
|
|
69
|
+
'->&b{}',
|
|
70
|
+
'->*b{}',
|
|
71
|
+
'->*b,c{}',
|
|
72
|
+
'->*b;c{}',
|
|
73
|
+
'->&b;c{}',
|
|
74
|
+
'->(*b){}',
|
|
75
|
+
'->(&b){}',
|
|
76
|
+
'->(*b,c){}',
|
|
77
|
+
'->(*b;c){}',
|
|
78
|
+
'->(&b;c){}',
|
|
55
79
|
]
|
|
56
80
|
|
|
57
81
|
EXPECT_1_METHOD=[
|
|
58
82
|
'def self.foo; 1 end',
|
|
59
|
-
'->{ foo=1 }
|
|
60
|
-
'->do foo=1 end
|
|
83
|
+
'->{ foo=1 } + foo',
|
|
84
|
+
'->do foo=1 end + foo',
|
|
61
85
|
'def __FILE__.foo; 1 end',
|
|
62
86
|
'def __LINE__.foo; 1 end',
|
|
63
87
|
'def a(b,*c,d) 1 end',
|
|
@@ -89,58 +113,58 @@ module Ruby1_9OneLiners
|
|
|
89
113
|
'def __ENCODING__.foo; 1 end',
|
|
90
114
|
'def Z::__ENCODING__; 342 end',
|
|
91
115
|
#'def Z::__ENCODING__.foo; 1 end', #oops, 2 methods here
|
|
92
|
-
'/\\(?<foo>bar)/ =~ "baz"
|
|
93
|
-
"/\\(?'foo'bar)/ =~ 'baz'
|
|
94
|
-
'/\\c(?<foo>bar)/ =~ "baz"
|
|
95
|
-
"/\\c(?'foo'bar)/ =~ 'baz'
|
|
96
|
-
'/\\C-(?<foo>bar)/ =~ "baz"
|
|
97
|
-
"/\\C-(?'foo'bar)/ =~ 'baz'
|
|
98
|
-
'/\\M-(?<foo>bar)/ =~ "baz"
|
|
99
|
-
"/\\M-(?'foo'bar)/ =~ 'baz'
|
|
100
|
-
'/\\\\\\(?<foo>bar)/ =~ "baz"
|
|
101
|
-
"/\\\\\\(?'foo'bar)/ =~ 'baz'
|
|
102
|
-
'/\\\\c\\(?<foo>bar)/ =~ "baz"
|
|
103
|
-
"/\\\\c\\(?'foo'bar)/ =~ 'baz'
|
|
104
|
-
'/(?#(?<foo>bar))/ =~ "baz"
|
|
105
|
-
"/(?#(?'foo'bar))/ =~ 'baz'
|
|
106
|
-
'/[(?<foo>bar)]/ =~ "baz"
|
|
107
|
-
"/[(?'foo'bar)]/ =~ 'baz'
|
|
108
|
-
'/[qwe&&[^ty](?<foo>bar)]/ =~ "baz"
|
|
109
|
-
"/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'
|
|
110
|
-
'/[qwe&&[^(?<foo>bar)]]/ =~ "baz"
|
|
111
|
-
"/[qwe&&[^(?'foo'bar)]]/ =~ 'baz'
|
|
112
|
-
'/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"
|
|
113
|
-
"/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'
|
|
114
|
-
'/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"
|
|
115
|
-
"/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'
|
|
116
|
-
'/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"
|
|
117
|
-
"/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'
|
|
116
|
+
'/\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
117
|
+
"/\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
118
|
+
'/\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
119
|
+
"/\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
120
|
+
'/\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
121
|
+
"/\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
122
|
+
'/\\M-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
123
|
+
"/\\M-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
124
|
+
'/\\\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
125
|
+
"/\\\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
126
|
+
'/\\\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
127
|
+
"/\\\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
128
|
+
'/(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
129
|
+
"/(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
130
|
+
'/[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
131
|
+
"/[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
132
|
+
'/[qwe&&[^ty](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
133
|
+
"/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
134
|
+
'/[qwe&&[^(?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
135
|
+
"/[qwe&&[^(?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
136
|
+
'/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
137
|
+
"/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
138
|
+
'/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
139
|
+
"/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
140
|
+
'/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz" && foo +1',
|
|
141
|
+
"/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz' && foo +1",
|
|
118
142
|
|
|
119
|
-
'/z\\(?<foo>bar)/ =~ "baz"
|
|
120
|
-
"/z\\(?'foo'bar)/ =~ 'baz'
|
|
121
|
-
'/z\\c(?<foo>bar)/ =~ "baz"
|
|
122
|
-
"/z\\c(?'foo'bar)/ =~ 'baz'
|
|
123
|
-
'/z\\C-(?<foo>bar)/ =~ "baz"
|
|
124
|
-
"/z\\C-(?'foo'bar)/ =~ 'baz'
|
|
125
|
-
'/z\\M-(?<foo>bar)/ =~ "baz"
|
|
126
|
-
"/z\\M-(?'foo'bar)/ =~ 'baz'
|
|
127
|
-
'/z\\\\\\(?<foo>bar)/ =~ "baz"
|
|
128
|
-
"/z\\\\\\(?'foo'bar)/ =~ 'baz'
|
|
129
|
-
'/z\\\\c\\(?<foo>bar)/ =~ "baz"
|
|
130
|
-
"/z\\\\c\\(?'foo'bar)/ =~ 'baz'
|
|
131
|
-
'/z(?#(?<foo>bar))/ =~ "baz"
|
|
132
|
-
"/z(?#(?'foo'bar))/ =~ 'baz'
|
|
133
|
-
'/z[(?<foo>bar)]/ =~ "baz"
|
|
134
|
-
"/z[(?'foo'bar)]/ =~ 'baz'
|
|
135
|
-
'/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz"
|
|
136
|
-
"/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'
|
|
137
|
-
'/z[qwe&&[^(?<foo>bar)]]/ =~ "baz"
|
|
138
|
-
"/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz'
|
|
139
|
-
'/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"
|
|
140
|
-
"/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'
|
|
141
|
-
'/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"
|
|
142
|
-
"/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'
|
|
143
|
-
'/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"
|
|
144
|
-
"/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'
|
|
143
|
+
'/z\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
144
|
+
"/z\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
145
|
+
'/z\\c(?<foo>bar)/ =~ "baz" && foo +1',
|
|
146
|
+
"/z\\c(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
147
|
+
'/z\\C-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
148
|
+
"/z\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
149
|
+
'/z\\M-(?<foo>bar)/ =~ "baz" && foo +1',
|
|
150
|
+
"/z\\M-(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
151
|
+
'/z\\\\\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
152
|
+
"/z\\\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
153
|
+
'/z\\\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
|
|
154
|
+
"/z\\\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
|
|
155
|
+
'/z(?#(?<foo>bar))/ =~ "baz" && foo +1',
|
|
156
|
+
"/z(?#(?'foo'bar))/ =~ 'baz' && foo +1",
|
|
157
|
+
'/z[(?<foo>bar)]/ =~ "baz" && foo +1',
|
|
158
|
+
"/z[(?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
159
|
+
'/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
160
|
+
"/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
161
|
+
'/z[qwe&&[^(?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
162
|
+
"/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
163
|
+
'/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz" && foo +1',
|
|
164
|
+
"/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz' && foo +1",
|
|
165
|
+
'/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz" && foo +1',
|
|
166
|
+
"/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz' && foo +1",
|
|
167
|
+
'/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz" && foo +1',
|
|
168
|
+
"/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz' && foo +1",
|
|
145
169
|
]
|
|
146
170
|
end
|
|
@@ -61,9 +61,6 @@ p %(1)
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
|
|
64
|
-
=begin
|
|
65
|
-
=end
|
|
66
|
-
|
|
67
64
|
p <<p
|
|
68
65
|
\
|
|
69
66
|
p
|
|
@@ -728,23 +725,10 @@ x {
|
|
|
728
725
|
p(p %(c))
|
|
729
726
|
}
|
|
730
727
|
|
|
731
|
-
=begin not supported yet
|
|
732
728
|
p "#{<<kekerz}#{"foob"
|
|
733
729
|
zimpler
|
|
734
730
|
kekerz
|
|
735
731
|
}"
|
|
736
|
-
=end
|
|
737
|
-
|
|
738
|
-
=begin not supported yet
|
|
739
|
-
aaa=<<whatnot; p "#{'uh,yeah'
|
|
740
|
-
gonna take it down, to the nitty-grit
|
|
741
|
-
gonna tell you mother-fuckers why you ain't shit
|
|
742
|
-
cause suckers like you just make me strong
|
|
743
|
-
you been pumpin' that bullshit all day long
|
|
744
|
-
whatnot
|
|
745
|
-
}"
|
|
746
|
-
p aaa
|
|
747
|
-
=end
|
|
748
732
|
|
|
749
733
|
#test variable creation in string inclusion
|
|
750
734
|
#currently broken because string inclusions
|
|
@@ -761,13 +745,11 @@ p %w[well, whaddaya know].map{|j| <<-END }
|
|
|
761
745
|
}45634543
|
|
762
746
|
END
|
|
763
747
|
|
|
764
|
-
=begin not supported yet
|
|
765
748
|
p "#{<<foobar3}"
|
|
766
749
|
bim
|
|
767
750
|
baz
|
|
768
751
|
bof
|
|
769
752
|
foobar3
|
|
770
|
-
=end
|
|
771
753
|
|
|
772
754
|
x do
|
|
773
755
|
a,b,* = [1,2,3,4,5,6,7,8]
|
|
@@ -947,25 +929,25 @@ p(/
|
|
|
947
929
|
p <<stuff+'foobar'.tr('j-l','d-f')
|
|
948
930
|
"more stuff"
|
|
949
931
|
12345678
|
|
950
|
-
the quick brown fox jumped over the lazy
|
|
932
|
+
the quick brown fox jumped over the lazy dog0
|
|
951
933
|
stuff
|
|
952
934
|
|
|
953
935
|
p <<stuff+'foobar'.tr('j-l','d-f')
|
|
954
936
|
"more stuff"
|
|
955
937
|
12345678
|
|
956
|
-
the quick brown fox jumped over the lazy
|
|
938
|
+
the quick brown fox jumped over the lazy dog1
|
|
957
939
|
stuff
|
|
958
940
|
|
|
959
941
|
p <<stuff+'foobar'.tr('j-l','d-f')\
|
|
960
942
|
+"more stuff"
|
|
961
943
|
12345678
|
|
962
|
-
the quick brown fox jumped over the lazy
|
|
944
|
+
the quick brown fox jumped over the lazy dog2
|
|
963
945
|
stuff
|
|
964
946
|
|
|
965
947
|
p <<stuff+'foobar'\
|
|
966
948
|
+"more stuff"
|
|
967
949
|
12345678
|
|
968
|
-
the quick brown fox jumped over the lazy
|
|
950
|
+
the quick brown fox jumped over the lazy dog3
|
|
969
951
|
stuff
|
|
970
952
|
|
|
971
953
|
|
|
@@ -1100,11 +1082,6 @@ for i in (begin
|
|
|
1100
1082
|
[44,55,66,77,88] end) do p i**Math.sqrt(i) end
|
|
1101
1083
|
}
|
|
1102
1084
|
|
|
1103
|
-
x{
|
|
1104
|
-
for i in if false then foob12345; else
|
|
1105
|
-
[44,55,66,77,88] end do p i**Math.sqrt(i) end
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
1085
|
x{
|
|
1109
1086
|
for i in (
|
|
1110
1087
|
[44,55,66,77,88]) do p i**Math.sqrt(i) end
|
|
@@ -1185,8 +1162,53 @@ sdsdfsdfs
|
|
|
1185
1162
|
A
|
|
1186
1163
|
|
|
1187
1164
|
|
|
1165
|
+
module
|
|
1166
|
+
=begin
|
|
1167
|
+
=end
|
|
1168
|
+
A; end
|
|
1169
|
+
|
|
1170
|
+
module
|
|
1171
|
+
=begin
|
|
1172
|
+
foo
|
|
1173
|
+
=end
|
|
1174
|
+
A; end
|
|
1175
|
+
|
|
1176
|
+
module
|
|
1177
|
+
=begin
|
|
1178
|
+
=end
|
|
1179
|
+
A::
|
|
1180
|
+
=begin
|
|
1181
|
+
=end
|
|
1182
|
+
B; end
|
|
1183
|
+
|
|
1184
|
+
module
|
|
1185
|
+
#=begin
|
|
1186
|
+
#=end
|
|
1187
|
+
A::
|
|
1188
|
+
#=begin
|
|
1189
|
+
#=end
|
|
1190
|
+
B; end
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
module
|
|
1194
|
+
=begin
|
|
1195
|
+
foo
|
|
1196
|
+
=end
|
|
1197
|
+
A::
|
|
1198
|
+
=begin
|
|
1199
|
+
bar
|
|
1200
|
+
=end
|
|
1201
|
+
B; end
|
|
1188
1202
|
|
|
1203
|
+
<<x.
|
|
1204
|
+
1111
|
|
1205
|
+
x
|
|
1206
|
+
delete()
|
|
1189
1207
|
|
|
1208
|
+
<<x::
|
|
1209
|
+
1111
|
|
1210
|
+
x
|
|
1211
|
+
delete()
|
|
1190
1212
|
|
|
1191
1213
|
\
|
|
1192
1214
|
__END__
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
module TestCases
|
|
2
2
|
# fail unless File.exist 'test/data/oneliners.rb' and File.exist 'test/data/stanzas.rb'
|
|
3
3
|
rldir=$:.find{|dir| File.exist? dir+'/rubylexer/test/oneliners.rb' and File.exist? dir+'/rubylexer/test/stanzas.rb' }
|
|
4
|
-
ONELINERS=IO.readlines(rldir+'/rubylexer/test/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s
|
|
4
|
+
ONELINERS=IO.readlines(rldir+'/rubylexer/test/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s]/).reverse
|
|
5
5
|
STANZAS=IO.read(rldir+'/rubylexer/test/stanzas.rb').split("\n\n").grep(/./).reverse
|
|
6
6
|
STANZAS.each{|stanza| stanza<<"\n" }
|
|
7
|
-
ILLEGAL_ONELINERS=IO.readlines(rldir+'/rubylexer/test/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s
|
|
7
|
+
ILLEGAL_ONELINERS=IO.readlines(rldir+'/rubylexer/test/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s]/).reverse
|
|
8
8
|
ILLEGAL_STANZAS=IO.read(rldir+'/rubylexer/test/illegal_stanzas.rb').split("\n\n").grep(/./).reverse
|
|
9
9
|
|
|
10
10
|
datadir=$:.find{|dir| File.exist? dir+'/../test/data/p.rb' }
|
data/lib/rubylexer/token.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
=begin
|
|
1
|
+
=begin
|
|
2
2
|
rubylexer - a ruby lexer written in ruby
|
|
3
|
-
Copyright (C) 2004,2005,2008 Caleb Clausen
|
|
3
|
+
Copyright (C) 2004,2005,2008, 2011 Caleb Clausen
|
|
4
4
|
|
|
5
5
|
This library is free software; you can redistribute it and/or
|
|
6
6
|
modify it under the terms of the GNU Lesser General Public
|
|
@@ -29,6 +29,7 @@ class Token
|
|
|
29
29
|
attr_accessor :offset #file offset of start of this token
|
|
30
30
|
attr_accessor :as #if defined, a KeywordToken which this token stands in for.
|
|
31
31
|
attr_accessor :allow_ooo_offset #hack
|
|
32
|
+
attr_accessor :endline
|
|
32
33
|
|
|
33
34
|
def initialize(ident,offset=nil)
|
|
34
35
|
@ident=ident
|
|
@@ -40,6 +41,29 @@ class Token
|
|
|
40
41
|
def has_no_block?; false end
|
|
41
42
|
|
|
42
43
|
attr_accessor :tag
|
|
44
|
+
|
|
45
|
+
attr_writer :startline
|
|
46
|
+
def startline
|
|
47
|
+
return @startline if defined? @startline
|
|
48
|
+
return endline
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def linecount; 0 end
|
|
52
|
+
|
|
53
|
+
alias orig_inspect inspect
|
|
54
|
+
alias dump inspect
|
|
55
|
+
|
|
56
|
+
#this is merely the normal definition of inspect
|
|
57
|
+
#and is unneeded in ruby 1.8
|
|
58
|
+
#but in 1.9, defining to_s seemingly overrides the built-in Object#inspect
|
|
59
|
+
#and you can't get it back, no matter what.
|
|
60
|
+
#fucking 1.9
|
|
61
|
+
def inspect
|
|
62
|
+
ivars=instance_variables.map{|ivar|
|
|
63
|
+
ivar.to_s+'='+instance_variable_get(ivar).inspect
|
|
64
|
+
}.join(' ')
|
|
65
|
+
%[#<#{self.class}: #{ivars}>]
|
|
66
|
+
end
|
|
43
67
|
end
|
|
44
68
|
|
|
45
69
|
#-------------------------
|
|
@@ -52,6 +76,17 @@ end
|
|
|
52
76
|
|
|
53
77
|
#-------------------------
|
|
54
78
|
class KeywordToken < WToken #also some operators
|
|
79
|
+
def initialize(*args)
|
|
80
|
+
if Hash===args.last
|
|
81
|
+
opts=args.pop
|
|
82
|
+
as=opts.delete :as
|
|
83
|
+
fail unless opts.empty?
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
super(*args)
|
|
87
|
+
self.as=as
|
|
88
|
+
end
|
|
89
|
+
|
|
55
90
|
|
|
56
91
|
#-----------------------------------
|
|
57
92
|
def set_callsite!(x=true)
|
|
@@ -109,8 +144,14 @@ end
|
|
|
109
144
|
|
|
110
145
|
#-------------------------
|
|
111
146
|
class OperatorToken < WToken
|
|
112
|
-
|
|
147
|
+
def initialize(*args)
|
|
148
|
+
@tag=nil
|
|
149
|
+
super
|
|
150
|
+
end
|
|
113
151
|
attr_writer :as
|
|
152
|
+
|
|
153
|
+
def unary= flag; @tag=:unary if flag end
|
|
154
|
+
def unary; @tag==:unary end
|
|
114
155
|
alias prefix? unary
|
|
115
156
|
def infix?; !prefix? end
|
|
116
157
|
|
|
@@ -149,8 +190,20 @@ end
|
|
|
149
190
|
|
|
150
191
|
#-------------------------
|
|
151
192
|
class NumberToken < Token
|
|
152
|
-
def to_s
|
|
153
|
-
|
|
193
|
+
def to_s
|
|
194
|
+
if defined? @char_literal and @char_literal
|
|
195
|
+
chr=@ident.chr
|
|
196
|
+
'?'+case chr
|
|
197
|
+
when " "; '\s'
|
|
198
|
+
when /[!-~]/; chr
|
|
199
|
+
else chr.inspect[1...-1]
|
|
200
|
+
end
|
|
201
|
+
else
|
|
202
|
+
@ident.to_s
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
def negative; /\A-/ === to_s end
|
|
206
|
+
attr_accessor :char_literal
|
|
154
207
|
end
|
|
155
208
|
|
|
156
209
|
#-------------------------
|
|
@@ -207,11 +260,18 @@ end
|
|
|
207
260
|
|
|
208
261
|
#-------------------------
|
|
209
262
|
class NewlineToken < Token
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
263
|
+
def initialize(nlstr="\n",offset=nil)
|
|
264
|
+
super(nlstr,offset)
|
|
265
|
+
#@char=''
|
|
266
|
+
end
|
|
267
|
+
def as; ';' end
|
|
268
|
+
|
|
269
|
+
def linecount; 1 end
|
|
270
|
+
|
|
271
|
+
def startline
|
|
272
|
+
@endline-1
|
|
273
|
+
end
|
|
274
|
+
def startline=bogus; end
|
|
215
275
|
end
|
|
216
276
|
|
|
217
277
|
#-------------------------
|
|
@@ -221,19 +281,31 @@ class StringToken < Token
|
|
|
221
281
|
attr_accessor :modifiers #for regex only
|
|
222
282
|
attr_accessor :elems
|
|
223
283
|
attr_accessor :startline
|
|
224
|
-
attr_accessor :line #line on which the string ENDS
|
|
225
284
|
attr_accessor :bs_handler
|
|
226
285
|
|
|
227
286
|
attr_accessor :open #exact sequence of chars used to start the str
|
|
228
287
|
attr_accessor :close #exact seq of (1) char to stop the str
|
|
229
288
|
|
|
230
289
|
attr_accessor :lvars #names used in named backrefs if this is a regex
|
|
290
|
+
|
|
291
|
+
def linecount; line-startline end
|
|
231
292
|
|
|
293
|
+
def utf8?
|
|
294
|
+
@utf8||=nil
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def utf8!
|
|
298
|
+
@utf8=true
|
|
299
|
+
end
|
|
300
|
+
|
|
232
301
|
def with_line(line)
|
|
233
|
-
@
|
|
302
|
+
@endline=line
|
|
234
303
|
self
|
|
235
304
|
end
|
|
236
305
|
|
|
306
|
+
def line; @endline end
|
|
307
|
+
def line= l; @endline=l end
|
|
308
|
+
|
|
237
309
|
def initialize(type='"',ident='')
|
|
238
310
|
super(ident)
|
|
239
311
|
type=="'" and type='"'
|
|
@@ -241,7 +313,7 @@ class StringToken < Token
|
|
|
241
313
|
assert @char[/^[\[{"`\/]$/] #"
|
|
242
314
|
@elems=[ident.dup] #why .dup?
|
|
243
315
|
@modifiers=nil
|
|
244
|
-
@
|
|
316
|
+
@endline=nil
|
|
245
317
|
end
|
|
246
318
|
|
|
247
319
|
DQUOTE_ESCAPE_TABLE = [
|
|
@@ -291,6 +363,8 @@ class StringToken < Token
|
|
|
291
363
|
result="%r"+result if RubyLexer::WHSPLF[result[1,1]]
|
|
292
364
|
end
|
|
293
365
|
|
|
366
|
+
result<<"\n" if open.empty? and result[-1] != ?\n
|
|
367
|
+
|
|
294
368
|
return result
|
|
295
369
|
end
|
|
296
370
|
|
|
@@ -362,9 +436,9 @@ private
|
|
|
362
436
|
UNESC_DELIMS={}
|
|
363
437
|
|
|
364
438
|
#simpler transform, preserves original exactly
|
|
365
|
-
def simple_transform(strfrag,starter,ender)
|
|
439
|
+
def simple_transform(strfrag,starter,ender) #appears to be unused
|
|
366
440
|
assert('[{/'[@char])
|
|
367
|
-
#strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='['
|
|
441
|
+
#strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='[' #esc #{
|
|
368
442
|
delimchars=Regexp.quote starter+ender
|
|
369
443
|
delimchars+=Regexp.quote("#") unless @char=='[' #escape beginning of string iterpolations
|
|
370
444
|
|
|
@@ -390,7 +464,7 @@ private
|
|
|
390
464
|
return strfrag
|
|
391
465
|
end
|
|
392
466
|
|
|
393
|
-
def transform(strfrag,starter,ender)
|
|
467
|
+
def transform(strfrag,starter,ender) #appears to be unused
|
|
394
468
|
strfrag.gsub!("\\",'\\'*4)
|
|
395
469
|
strfrag.gsub!(/#([{$@])/,'\\#\\1')
|
|
396
470
|
strfrag.gsub!(Regexp.new("[\\"+starter+"\\"+ender+"]"),'\\\\\\&') unless @char=='?'
|
|
@@ -427,8 +501,9 @@ private
|
|
|
427
501
|
end
|
|
428
502
|
|
|
429
503
|
#-------------------------
|
|
430
|
-
class RenderExactlyStringToken < StringToken
|
|
504
|
+
class RenderExactlyStringToken < StringToken
|
|
431
505
|
alias transform simple_transform
|
|
506
|
+
#transform isn't called anymore, so there's no need for this hacky class
|
|
432
507
|
end
|
|
433
508
|
|
|
434
509
|
#-------------------------
|
|
@@ -471,9 +546,13 @@ class HerePlaceholderToken < WToken
|
|
|
471
546
|
|
|
472
547
|
#def with_line(line) @string.line=line; self end
|
|
473
548
|
|
|
474
|
-
def line; @string.line end
|
|
475
|
-
def line=line; @
|
|
549
|
+
def line; @line || @string.line end
|
|
550
|
+
def line=line; @line=line end
|
|
476
551
|
|
|
552
|
+
def startline; @line end
|
|
553
|
+
alias endline startline
|
|
554
|
+
def startline=x; end
|
|
555
|
+
alias endline= startline=
|
|
477
556
|
end
|
|
478
557
|
|
|
479
558
|
#-------------------------
|
|
@@ -484,6 +563,13 @@ end
|
|
|
484
563
|
#-------------------------
|
|
485
564
|
class IgnoreToken < Token
|
|
486
565
|
include StillIgnoreToken
|
|
566
|
+
|
|
567
|
+
def initialize(ident,*stuff)
|
|
568
|
+
@linecount=ident.count "\n"
|
|
569
|
+
super
|
|
570
|
+
end
|
|
571
|
+
|
|
572
|
+
attr :linecount
|
|
487
573
|
end
|
|
488
574
|
|
|
489
575
|
#-------------------------
|
|
@@ -511,6 +597,23 @@ class NoWsToken < ZwToken
|
|
|
511
597
|
end
|
|
512
598
|
end
|
|
513
599
|
|
|
600
|
+
#-------------------------
|
|
601
|
+
class ShebangToken < IgnoreToken
|
|
602
|
+
def initialize(text)
|
|
603
|
+
super text,0
|
|
604
|
+
end
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
#-------------------------
|
|
608
|
+
class EncodingDeclToken < IgnoreToken
|
|
609
|
+
def initialize(text,encoding,offset)
|
|
610
|
+
text||=''
|
|
611
|
+
super text,offset
|
|
612
|
+
@encoding=encoding
|
|
613
|
+
end
|
|
614
|
+
attr :encoding
|
|
615
|
+
end
|
|
616
|
+
|
|
514
617
|
#-------------------------
|
|
515
618
|
class ImplicitParamListStartToken < KeywordToken
|
|
516
619
|
include StillIgnoreToken
|
|
@@ -579,6 +682,19 @@ class EscNlToken < IgnoreToken
|
|
|
579
682
|
@filename=filename
|
|
580
683
|
@linenum=linenum
|
|
581
684
|
end
|
|
685
|
+
|
|
686
|
+
attr_accessor :filename,:linenum
|
|
687
|
+
|
|
688
|
+
def linecount; 1 end
|
|
689
|
+
|
|
690
|
+
def startline
|
|
691
|
+
@linenum-1
|
|
692
|
+
end
|
|
693
|
+
def endline
|
|
694
|
+
@linenum
|
|
695
|
+
end
|
|
696
|
+
def startline= bogus; end
|
|
697
|
+
alias endline= linenum=
|
|
582
698
|
end
|
|
583
699
|
|
|
584
700
|
#-------------------------
|
|
@@ -586,9 +702,10 @@ class EoiToken < Token
|
|
|
586
702
|
attr :file
|
|
587
703
|
alias :pos :offset
|
|
588
704
|
|
|
589
|
-
def initialize(cause,file, offset=nil)
|
|
705
|
+
def initialize(cause,file, offset=nil,line=nil)
|
|
590
706
|
super(cause,offset)
|
|
591
707
|
@file=file
|
|
708
|
+
@endline=line
|
|
592
709
|
end
|
|
593
710
|
end
|
|
594
711
|
|
|
@@ -598,7 +715,7 @@ class HereBodyToken < IgnoreToken
|
|
|
598
715
|
attr_accessor :open,:close
|
|
599
716
|
def initialize(headtok,linecount)
|
|
600
717
|
assert HerePlaceholderToken===headtok
|
|
601
|
-
|
|
718
|
+
@ident,@offset=headtok.string,headtok.string.offset
|
|
602
719
|
@headtok=headtok
|
|
603
720
|
@linecount=linecount
|
|
604
721
|
end
|
|
@@ -606,6 +723,14 @@ class HereBodyToken < IgnoreToken
|
|
|
606
723
|
def line
|
|
607
724
|
@ident.line
|
|
608
725
|
end
|
|
726
|
+
alias endline line
|
|
727
|
+
def endline= line
|
|
728
|
+
@ident.line= line
|
|
729
|
+
end
|
|
730
|
+
|
|
731
|
+
def startline
|
|
732
|
+
line-@linecount+1
|
|
733
|
+
end
|
|
609
734
|
|
|
610
735
|
def to_s
|
|
611
736
|
@ident.to_s
|
|
@@ -629,15 +754,20 @@ class FileAndLineToken < IgnoreToken
|
|
|
629
754
|
#def char; '#' end
|
|
630
755
|
|
|
631
756
|
def to_s()
|
|
632
|
-
[
|
|
757
|
+
%[##@ident:#@line]
|
|
633
758
|
end
|
|
634
759
|
|
|
635
760
|
def file() @ident end
|
|
636
761
|
def subitem() @line end #needed?
|
|
762
|
+
|
|
763
|
+
def endline; @line end
|
|
764
|
+
def startline; @line end
|
|
765
|
+
alias endline= line=
|
|
766
|
+
def startline= bogus; end
|
|
637
767
|
end
|
|
638
768
|
|
|
639
769
|
#-------------------------
|
|
640
|
-
class OutlinedHereBodyToken < HereBodyToken
|
|
770
|
+
class OutlinedHereBodyToken < HereBodyToken #appears to be unused
|
|
641
771
|
def to_s
|
|
642
772
|
assert HerePlaceholderToken===@headtok
|
|
643
773
|
result=@headtok.string
|