rubylexer 0.7.7 → 0.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +4 -0
  2. data/History.txt +64 -0
  3. data/Makefile +2 -2
  4. data/README.txt +13 -9
  5. data/bin/rubylexer +113 -0
  6. data/lib/assert.rb +1 -1
  7. data/lib/rubylexer.rb +856 -305
  8. data/lib/rubylexer/charhandler.rb +1 -1
  9. data/lib/rubylexer/charset.rb +15 -7
  10. data/lib/rubylexer/context.rb +10 -2
  11. data/lib/rubylexer/lextable.rb +1 -0
  12. data/lib/rubylexer/rubycode.rb +1 -1
  13. data/lib/rubylexer/rulexer.rb +106 -32
  14. data/lib/rubylexer/symboltable.rb +1 -1
  15. data/lib/rubylexer/test/oneliners.rb +15 -5
  16. data/lib/rubylexer/test/oneliners_1.9.rb +116 -92
  17. data/lib/rubylexer/test/stanzas.rb +49 -27
  18. data/lib/rubylexer/test/testcases.rb +2 -2
  19. data/lib/rubylexer/token.rb +153 -23
  20. data/lib/rubylexer/tokenprinter.rb +9 -6
  21. data/lib/rubylexer/version.rb +1 -1
  22. data/rubylexer.gemspec +12 -8
  23. data/test/bad/ruby_lexer.rb +7 -0
  24. data/test/code/deletewarns.rb +1 -1
  25. data/test/code/dumptokens.rb +1 -81
  26. data/test/code/heredoc_blast_test.rb +112 -0
  27. data/test/code/locatetest.rb +1 -1
  28. data/test/code/regression.rb +23 -23
  29. data/test/code/rubylexervsruby.rb +59 -12
  30. data/test/code/tokentest.rb +62 -52
  31. data/test/data/23.rb +0 -1
  32. data/test/data/g.rb +0 -1
  33. data/test/data/heremonsters.rb +1 -1
  34. data/test/data/heremonsters_dos.rb +1 -1
  35. data/test/data/pre.rb +0 -1
  36. data/test/data/pre.unix.rb +0 -1
  37. data/test/data/putstext.rb +4 -0
  38. data/test/data/regtest.rb +0 -1
  39. data/test/data/stuffydog.rb +5 -0
  40. data/test/data/stuffydog2.rb +5 -0
  41. data/test/data/wsdlDriver.rb +0 -1
  42. data/test/test.sh +1 -1
  43. data/test/test_all.rb +3 -0
  44. data/test/test_bad_rubylexer.rb +16 -0
  45. data/test/test_rubylexer_bad.rb +12 -0
  46. data/testing.txt +40 -20
  47. metadata +51 -38
@@ -1,5 +1,47 @@
1
+ #encoding: utf-8
2
+
1
3
  module Ruby1_9OneLiners
2
4
  EXPECT_NO_METHODS=[ #no errors either
5
+ '__ENCODING__',
6
+ '__ENCODING__ +"foo"',
7
+ 'module __ENCODING__::A; end',
8
+ '/(?<foo>bar)/ =~ "baz" && foo +1',
9
+ "/(?'foo'bar)/ =~ 'baz' && foo +1",
10
+ '/\\\\(?<foo>bar)/ =~ "baz" && foo +1',
11
+ "/\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
12
+ '/\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
13
+ "/\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
14
+ '/\\\\c(?<foo>bar)/ =~ "baz" && foo +1',
15
+ "/\\\\c(?'foo'bar)/ =~ 'baz' && foo +1",
16
+ '/\\C-\\(?<foo>bar)/ =~ "baz" && foo +1',
17
+ "/\\C-\\(?'foo'bar)/ =~ 'baz' && foo +1",
18
+ '/\\\\C-(?<foo>bar)/ =~ "baz" && foo +1',
19
+ "/\\\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
20
+ '/\\(?#(?<foo>bar))/ =~ "baz" && foo +1',
21
+ "/\\(?#(?'foo'bar))/ =~ 'baz' && foo +1",
22
+ '/\\[(?<foo>bar)]/ =~ "baz" && foo +1',
23
+ "/\\[(?'foo'bar)]/ =~ 'baz' && foo +1",
24
+
25
+ '/z(?<foo>bar)/ =~ "baz" && foo +1',
26
+ "/z(?'foo'bar)/ =~ 'baz' && foo +1",
27
+ '/z\\\\(?<foo>bar)/ =~ "baz" && foo +1',
28
+ "/z\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
29
+ '/z\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
30
+ "/z\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
31
+ '/z\\\\c(?<foo>bar)/ =~ "baz" && foo +1',
32
+ "/z\\\\c(?'foo'bar)/ =~ 'baz' && foo +1",
33
+ '/z\\C-\\(?<foo>bar)/ =~ "baz" && foo +1',
34
+ "/z\\C-\\(?'foo'bar)/ =~ 'baz' && foo +1",
35
+ '/z\\\\C-(?<foo>bar)/ =~ "baz" && foo +1',
36
+ "/z\\\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
37
+ '/z\\(?#(?<foo>bar))/ =~ "baz" && foo +1',
38
+ "/z\\(?#(?'foo'bar))/ =~ 'baz' && foo +1",
39
+ '/z\\[(?<foo>bar)]/ =~ "baz" && foo +1',
40
+ "/z\\[(?'foo'bar)]/ =~ 'baz' && foo +1",
41
+ "?\\uaeee",
42
+ "#encoding: utf-8\n ?♥", #utf8 heart char
43
+ '$f.($x,$y)',
44
+ '$f::($x,$y)',
3
45
  '->a; h do 123 end',
4
46
  '->{}',
5
47
  '-> {}',
@@ -14,50 +56,32 @@ module Ruby1_9OneLiners
14
56
  '->a,b;c{}',
15
57
  '->(a,b;){}',
16
58
  '->(a,b;c){}',
17
- '$f.($x,$y)',
18
- '$f::($x,$y)',
19
- '__ENCODING__',
20
- '__ENCODING__ +"foo"',
21
- 'module __ENCODING__::A; end',
22
- '/(?<foo>bar)/ =~ "baz"; foo +1',
23
- "/(?'foo'bar)/ =~ 'baz'; foo +1",
24
- '/\\\\(?<foo>bar)/ =~ "baz"; foo +1',
25
- "/\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
26
- '/\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
27
- "/\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
28
- '/\\\\c(?<foo>bar)/ =~ "baz"; foo +1',
29
- "/\\\\c(?'foo'bar)/ =~ 'baz'; foo +1",
30
- '/\\C-\\(?<foo>bar)/ =~ "baz"; foo +1',
31
- "/\\C-\\(?'foo'bar)/ =~ 'baz'; foo +1",
32
- '/\\\\C-(?<foo>bar)/ =~ "baz"; foo +1',
33
- "/\\\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
34
- '/\\(?#(?<foo>bar))/ =~ "baz"; foo +1',
35
- "/\\(?#(?'foo'bar))/ =~ 'baz'; foo +1",
36
- '/\\[(?<foo>bar)]/ =~ "baz"; foo +1',
37
- "/\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
38
-
39
- '/z(?<foo>bar)/ =~ "baz"; foo +1',
40
- "/z(?'foo'bar)/ =~ 'baz'; foo +1",
41
- '/z\\\\(?<foo>bar)/ =~ "baz"; foo +1',
42
- "/z\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
43
- '/z\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
44
- "/z\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
45
- '/z\\\\c(?<foo>bar)/ =~ "baz"; foo +1',
46
- "/z\\\\c(?'foo'bar)/ =~ 'baz'; foo +1",
47
- '/z\\C-\\(?<foo>bar)/ =~ "baz"; foo +1',
48
- "/z\\C-\\(?'foo'bar)/ =~ 'baz'; foo +1",
49
- '/z\\\\C-(?<foo>bar)/ =~ "baz"; foo +1',
50
- "/z\\\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
51
- '/z\\(?#(?<foo>bar))/ =~ "baz"; foo +1',
52
- "/z\\(?#(?'foo'bar))/ =~ 'baz'; foo +1",
53
- '/z\\[(?<foo>bar)]/ =~ "baz"; foo +1',
54
- "/z\\[(?'foo'bar)]/ =~ 'baz'; foo +1",
59
+ '->a,&b{}',
60
+ '->a,*b{}',
61
+ '->a,*b,c{}',
62
+ '->a,*b;c{}',
63
+ '->a,&b;c{}',
64
+ '->(a,*b){}',
65
+ '->(a,&b){}',
66
+ '->(a,*b,c){}',
67
+ '->(a,*b;c){}',
68
+ '->(a,&b;c){}',
69
+ '->&b{}',
70
+ '->*b{}',
71
+ '->*b,c{}',
72
+ '->*b;c{}',
73
+ '->&b;c{}',
74
+ '->(*b){}',
75
+ '->(&b){}',
76
+ '->(*b,c){}',
77
+ '->(*b;c){}',
78
+ '->(&b;c){}',
55
79
  ]
56
80
 
57
81
  EXPECT_1_METHOD=[
58
82
  'def self.foo; 1 end',
59
- '->{ foo=1 }; foo',
60
- '->do foo=1 end; foo',
83
+ '->{ foo=1 } + foo',
84
+ '->do foo=1 end + foo',
61
85
  'def __FILE__.foo; 1 end',
62
86
  'def __LINE__.foo; 1 end',
63
87
  'def a(b,*c,d) 1 end',
@@ -89,58 +113,58 @@ module Ruby1_9OneLiners
89
113
  'def __ENCODING__.foo; 1 end',
90
114
  'def Z::__ENCODING__; 342 end',
91
115
  #'def Z::__ENCODING__.foo; 1 end', #oops, 2 methods here
92
- '/\\(?<foo>bar)/ =~ "baz"; foo +1',
93
- "/\\(?'foo'bar)/ =~ 'baz'; foo +1",
94
- '/\\c(?<foo>bar)/ =~ "baz"; foo +1',
95
- "/\\c(?'foo'bar)/ =~ 'baz'; foo +1",
96
- '/\\C-(?<foo>bar)/ =~ "baz"; foo +1',
97
- "/\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
98
- '/\\M-(?<foo>bar)/ =~ "baz"; foo +1',
99
- "/\\M-(?'foo'bar)/ =~ 'baz'; foo +1",
100
- '/\\\\\\(?<foo>bar)/ =~ "baz"; foo +1',
101
- "/\\\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
102
- '/\\\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
103
- "/\\\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
104
- '/(?#(?<foo>bar))/ =~ "baz"; foo +1',
105
- "/(?#(?'foo'bar))/ =~ 'baz'; foo +1",
106
- '/[(?<foo>bar)]/ =~ "baz"; foo +1',
107
- "/[(?'foo'bar)]/ =~ 'baz'; foo +1",
108
- '/[qwe&&[^ty](?<foo>bar)]/ =~ "baz"; foo +1',
109
- "/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'; foo +1",
110
- '/[qwe&&[^(?<foo>bar)]]/ =~ "baz"; foo +1',
111
- "/[qwe&&[^(?'foo'bar)]]/ =~ 'baz'; foo +1",
112
- '/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"; foo +1',
113
- "/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'; foo +1",
114
- '/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"; foo +1',
115
- "/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'; foo +1",
116
- '/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"; foo +1',
117
- "/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'; foo +1",
116
+ '/\\(?<foo>bar)/ =~ "baz" && foo +1',
117
+ "/\\(?'foo'bar)/ =~ 'baz' && foo +1",
118
+ '/\\c(?<foo>bar)/ =~ "baz" && foo +1',
119
+ "/\\c(?'foo'bar)/ =~ 'baz' && foo +1",
120
+ '/\\C-(?<foo>bar)/ =~ "baz" && foo +1',
121
+ "/\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
122
+ '/\\M-(?<foo>bar)/ =~ "baz" && foo +1',
123
+ "/\\M-(?'foo'bar)/ =~ 'baz' && foo +1",
124
+ '/\\\\\\(?<foo>bar)/ =~ "baz" && foo +1',
125
+ "/\\\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
126
+ '/\\\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
127
+ "/\\\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
128
+ '/(?#(?<foo>bar))/ =~ "baz" && foo +1',
129
+ "/(?#(?'foo'bar))/ =~ 'baz' && foo +1",
130
+ '/[(?<foo>bar)]/ =~ "baz" && foo +1',
131
+ "/[(?'foo'bar)]/ =~ 'baz' && foo +1",
132
+ '/[qwe&&[^ty](?<foo>bar)]/ =~ "baz" && foo +1',
133
+ "/[qwe&&[^ty](?'foo'bar)]/ =~ 'baz' && foo +1",
134
+ '/[qwe&&[^(?<foo>bar)]]/ =~ "baz" && foo +1',
135
+ "/[qwe&&[^(?'foo'bar)]]/ =~ 'baz' && foo +1",
136
+ '/[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz" && foo +1',
137
+ "/[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz' && foo +1",
138
+ '/[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz" && foo +1',
139
+ "/[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz' && foo +1",
140
+ '/[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz" && foo +1',
141
+ "/[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz' && foo +1",
118
142
 
119
- '/z\\(?<foo>bar)/ =~ "baz"; foo +1',
120
- "/z\\(?'foo'bar)/ =~ 'baz'; foo +1",
121
- '/z\\c(?<foo>bar)/ =~ "baz"; foo +1',
122
- "/z\\c(?'foo'bar)/ =~ 'baz'; foo +1",
123
- '/z\\C-(?<foo>bar)/ =~ "baz"; foo +1',
124
- "/z\\C-(?'foo'bar)/ =~ 'baz'; foo +1",
125
- '/z\\M-(?<foo>bar)/ =~ "baz"; foo +1',
126
- "/z\\M-(?'foo'bar)/ =~ 'baz'; foo +1",
127
- '/z\\\\\\(?<foo>bar)/ =~ "baz"; foo +1',
128
- "/z\\\\\\(?'foo'bar)/ =~ 'baz'; foo +1",
129
- '/z\\\\c\\(?<foo>bar)/ =~ "baz"; foo +1',
130
- "/z\\\\c\\(?'foo'bar)/ =~ 'baz'; foo +1",
131
- '/z(?#(?<foo>bar))/ =~ "baz"; foo +1',
132
- "/z(?#(?'foo'bar))/ =~ 'baz'; foo +1",
133
- '/z[(?<foo>bar)]/ =~ "baz"; foo +1',
134
- "/z[(?'foo'bar)]/ =~ 'baz'; foo +1",
135
- '/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz"; foo +1',
136
- "/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz'; foo +1",
137
- '/z[qwe&&[^(?<foo>bar)]]/ =~ "baz"; foo +1',
138
- "/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz'; foo +1",
139
- '/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz"; foo +1',
140
- "/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz'; foo +1",
141
- '/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz"; foo +1',
142
- "/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz'; foo +1",
143
- '/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz"; foo +1',
144
- "/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz'; foo +1",
143
+ '/z\\(?<foo>bar)/ =~ "baz" && foo +1',
144
+ "/z\\(?'foo'bar)/ =~ 'baz' && foo +1",
145
+ '/z\\c(?<foo>bar)/ =~ "baz" && foo +1',
146
+ "/z\\c(?'foo'bar)/ =~ 'baz' && foo +1",
147
+ '/z\\C-(?<foo>bar)/ =~ "baz" && foo +1',
148
+ "/z\\C-(?'foo'bar)/ =~ 'baz' && foo +1",
149
+ '/z\\M-(?<foo>bar)/ =~ "baz" && foo +1',
150
+ "/z\\M-(?'foo'bar)/ =~ 'baz' && foo +1",
151
+ '/z\\\\\\(?<foo>bar)/ =~ "baz" && foo +1',
152
+ "/z\\\\\\(?'foo'bar)/ =~ 'baz' && foo +1",
153
+ '/z\\\\c\\(?<foo>bar)/ =~ "baz" && foo +1',
154
+ "/z\\\\c\\(?'foo'bar)/ =~ 'baz' && foo +1",
155
+ '/z(?#(?<foo>bar))/ =~ "baz" && foo +1',
156
+ "/z(?#(?'foo'bar))/ =~ 'baz' && foo +1",
157
+ '/z[(?<foo>bar)]/ =~ "baz" && foo +1',
158
+ "/z[(?'foo'bar)]/ =~ 'baz' && foo +1",
159
+ '/z[qwe&&[^ty](?<foo>bar)]/ =~ "baz" && foo +1',
160
+ "/z[qwe&&[^ty](?'foo'bar)]/ =~ 'baz' && foo +1",
161
+ '/z[qwe&&[^(?<foo>bar)]]/ =~ "baz" && foo +1',
162
+ "/z[qwe&&[^(?'foo'bar)]]/ =~ 'baz' && foo +1",
163
+ '/z[qwe&&[^ty&&[^uip]](?<foo>bar)]/ =~ "baz" && foo +1',
164
+ "/z[qwe&&[^ty&&[^uip]](?'foo'bar)]/ =~ 'baz' && foo +1",
165
+ '/z[qwe&&[^ty&&[^uip](?<foo>bar)]]/ =~ "baz" && foo +1',
166
+ "/z[qwe&&[^ty&&[^uip](?'foo'bar)]]/ =~ 'baz' && foo +1",
167
+ '/z[qwe&&[^ty&&[^uip(?<foo>bar)]]]/ =~ "baz" && foo +1',
168
+ "/z[qwe&&[^ty&&[^uip(?'foo'bar)]]]/ =~ 'baz' && foo +1",
145
169
  ]
146
170
  end
@@ -61,9 +61,6 @@ p %(1)
61
61
 
62
62
 
63
63
 
64
- =begin
65
- =end
66
-
67
64
  p <<p
68
65
  \
69
66
  p
@@ -728,23 +725,10 @@ x {
728
725
  p(p %(c))
729
726
  }
730
727
 
731
- =begin not supported yet
732
728
  p "#{<<kekerz}#{"foob"
733
729
  zimpler
734
730
  kekerz
735
731
  }"
736
- =end
737
-
738
- =begin not supported yet
739
- aaa=<<whatnot; p "#{'uh,yeah'
740
- gonna take it down, to the nitty-grit
741
- gonna tell you mother-fuckers why you ain't shit
742
- cause suckers like you just make me strong
743
- you been pumpin' that bullshit all day long
744
- whatnot
745
- }"
746
- p aaa
747
- =end
748
732
 
749
733
  #test variable creation in string inclusion
750
734
  #currently broken because string inclusions
@@ -761,13 +745,11 @@ p %w[well, whaddaya know].map{|j| <<-END }
761
745
  }45634543
762
746
  END
763
747
 
764
- =begin not supported yet
765
748
  p "#{<<foobar3}"
766
749
  bim
767
750
  baz
768
751
  bof
769
752
  foobar3
770
- =end
771
753
 
772
754
  x do
773
755
  a,b,* = [1,2,3,4,5,6,7,8]
@@ -947,25 +929,25 @@ p(/
947
929
  p <<stuff+'foobar'.tr('j-l','d-f')
948
930
  "more stuff"
949
931
  12345678
950
- the quick brown fox jumped over the lazy dog
932
+ the quick brown fox jumped over the lazy dog0
951
933
  stuff
952
934
 
953
935
  p <<stuff+'foobar'.tr('j-l','d-f')
954
936
  "more stuff"
955
937
  12345678
956
- the quick brown fox jumped over the lazy dog
938
+ the quick brown fox jumped over the lazy dog1
957
939
  stuff
958
940
 
959
941
  p <<stuff+'foobar'.tr('j-l','d-f')\
960
942
  +"more stuff"
961
943
  12345678
962
- the quick brown fox jumped over the lazy dog
944
+ the quick brown fox jumped over the lazy dog2
963
945
  stuff
964
946
 
965
947
  p <<stuff+'foobar'\
966
948
  +"more stuff"
967
949
  12345678
968
- the quick brown fox jumped over the lazy dog
950
+ the quick brown fox jumped over the lazy dog3
969
951
  stuff
970
952
 
971
953
 
@@ -1100,11 +1082,6 @@ for i in (begin
1100
1082
  [44,55,66,77,88] end) do p i**Math.sqrt(i) end
1101
1083
  }
1102
1084
 
1103
- x{
1104
- for i in if false then foob12345; else
1105
- [44,55,66,77,88] end do p i**Math.sqrt(i) end
1106
- }
1107
-
1108
1085
  x{
1109
1086
  for i in (
1110
1087
  [44,55,66,77,88]) do p i**Math.sqrt(i) end
@@ -1185,8 +1162,53 @@ sdsdfsdfs
1185
1162
  A
1186
1163
 
1187
1164
 
1165
+ module
1166
+ =begin
1167
+ =end
1168
+ A; end
1169
+
1170
+ module
1171
+ =begin
1172
+ foo
1173
+ =end
1174
+ A; end
1175
+
1176
+ module
1177
+ =begin
1178
+ =end
1179
+ A::
1180
+ =begin
1181
+ =end
1182
+ B; end
1183
+
1184
+ module
1185
+ #=begin
1186
+ #=end
1187
+ A::
1188
+ #=begin
1189
+ #=end
1190
+ B; end
1191
+
1192
+
1193
+ module
1194
+ =begin
1195
+ foo
1196
+ =end
1197
+ A::
1198
+ =begin
1199
+ bar
1200
+ =end
1201
+ B; end
1188
1202
 
1203
+ <<x.
1204
+ 1111
1205
+ x
1206
+ delete()
1189
1207
 
1208
+ <<x::
1209
+ 1111
1210
+ x
1211
+ delete()
1190
1212
 
1191
1213
  \
1192
1214
  __END__
@@ -1,10 +1,10 @@
1
1
  module TestCases
2
2
  # fail unless File.exist 'test/data/oneliners.rb' and File.exist 'test/data/stanzas.rb'
3
3
  rldir=$:.find{|dir| File.exist? dir+'/rubylexer/test/oneliners.rb' and File.exist? dir+'/rubylexer/test/stanzas.rb' }
4
- ONELINERS=IO.readlines(rldir+'/rubylexer/test/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s\n]/).reverse
4
+ ONELINERS=IO.readlines(rldir+'/rubylexer/test/oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s]/).reverse
5
5
  STANZAS=IO.read(rldir+'/rubylexer/test/stanzas.rb').split("\n\n").grep(/./).reverse
6
6
  STANZAS.each{|stanza| stanza<<"\n" }
7
- ILLEGAL_ONELINERS=IO.readlines(rldir+'/rubylexer/test/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s\n]/).reverse
7
+ ILLEGAL_ONELINERS=IO.readlines(rldir+'/rubylexer/test/illegal_oneliners.rb').map{|x| x.chomp}.grep(/\A\s*[^#\s]/).reverse
8
8
  ILLEGAL_STANZAS=IO.read(rldir+'/rubylexer/test/illegal_stanzas.rb').split("\n\n").grep(/./).reverse
9
9
 
10
10
  datadir=$:.find{|dir| File.exist? dir+'/../test/data/p.rb' }
@@ -1,6 +1,6 @@
1
- =begin legal crap
1
+ =begin
2
2
  rubylexer - a ruby lexer written in ruby
3
- Copyright (C) 2004,2005,2008 Caleb Clausen
3
+ Copyright (C) 2004,2005,2008, 2011 Caleb Clausen
4
4
 
5
5
  This library is free software; you can redistribute it and/or
6
6
  modify it under the terms of the GNU Lesser General Public
@@ -29,6 +29,7 @@ class Token
29
29
  attr_accessor :offset #file offset of start of this token
30
30
  attr_accessor :as #if defined, a KeywordToken which this token stands in for.
31
31
  attr_accessor :allow_ooo_offset #hack
32
+ attr_accessor :endline
32
33
 
33
34
  def initialize(ident,offset=nil)
34
35
  @ident=ident
@@ -40,6 +41,29 @@ class Token
40
41
  def has_no_block?; false end
41
42
 
42
43
  attr_accessor :tag
44
+
45
+ attr_writer :startline
46
+ def startline
47
+ return @startline if defined? @startline
48
+ return endline
49
+ end
50
+
51
+ def linecount; 0 end
52
+
53
+ alias orig_inspect inspect
54
+ alias dump inspect
55
+
56
+ #this is merely the normal definition of inspect
57
+ #and is unneeded in ruby 1.8
58
+ #but in 1.9, defining to_s seemingly overrides the built-in Object#inspect
59
+ #and you can't get it back, no matter what.
60
+ #fucking 1.9
61
+ def inspect
62
+ ivars=instance_variables.map{|ivar|
63
+ ivar.to_s+'='+instance_variable_get(ivar).inspect
64
+ }.join(' ')
65
+ %[#<#{self.class}: #{ivars}>]
66
+ end
43
67
  end
44
68
 
45
69
  #-------------------------
@@ -52,6 +76,17 @@ end
52
76
 
53
77
  #-------------------------
54
78
  class KeywordToken < WToken #also some operators
79
+ def initialize(*args)
80
+ if Hash===args.last
81
+ opts=args.pop
82
+ as=opts.delete :as
83
+ fail unless opts.empty?
84
+ end
85
+
86
+ super(*args)
87
+ self.as=as
88
+ end
89
+
55
90
 
56
91
  #-----------------------------------
57
92
  def set_callsite!(x=true)
@@ -109,8 +144,14 @@ end
109
144
 
110
145
  #-------------------------
111
146
  class OperatorToken < WToken
112
- attr_accessor :unary
147
+ def initialize(*args)
148
+ @tag=nil
149
+ super
150
+ end
113
151
  attr_writer :as
152
+
153
+ def unary= flag; @tag=:unary if flag end
154
+ def unary; @tag==:unary end
114
155
  alias prefix? unary
115
156
  def infix?; !prefix? end
116
157
 
@@ -149,8 +190,20 @@ end
149
190
 
150
191
  #-------------------------
151
192
  class NumberToken < Token
152
- def to_s; @ident.to_s end
153
- def negative; /\A-/ === ident end
193
+ def to_s
194
+ if defined? @char_literal and @char_literal
195
+ chr=@ident.chr
196
+ '?'+case chr
197
+ when " "; '\s'
198
+ when /[!-~]/; chr
199
+ else chr.inspect[1...-1]
200
+ end
201
+ else
202
+ @ident.to_s
203
+ end
204
+ end
205
+ def negative; /\A-/ === to_s end
206
+ attr_accessor :char_literal
154
207
  end
155
208
 
156
209
  #-------------------------
@@ -207,11 +260,18 @@ end
207
260
 
208
261
  #-------------------------
209
262
  class NewlineToken < Token
210
- def initialize(nlstr="\n",offset=nil)
211
- super(nlstr,offset)
212
- #@char=''
213
- end
214
- def as; ';' end
263
+ def initialize(nlstr="\n",offset=nil)
264
+ super(nlstr,offset)
265
+ #@char=''
266
+ end
267
+ def as; ';' end
268
+
269
+ def linecount; 1 end
270
+
271
+ def startline
272
+ @endline-1
273
+ end
274
+ def startline=bogus; end
215
275
  end
216
276
 
217
277
  #-------------------------
@@ -221,19 +281,31 @@ class StringToken < Token
221
281
  attr_accessor :modifiers #for regex only
222
282
  attr_accessor :elems
223
283
  attr_accessor :startline
224
- attr_accessor :line #line on which the string ENDS
225
284
  attr_accessor :bs_handler
226
285
 
227
286
  attr_accessor :open #exact sequence of chars used to start the str
228
287
  attr_accessor :close #exact seq of (1) char to stop the str
229
288
 
230
289
  attr_accessor :lvars #names used in named backrefs if this is a regex
290
+
291
+ def linecount; line-startline end
231
292
 
293
+ def utf8?
294
+ @utf8||=nil
295
+ end
296
+
297
+ def utf8!
298
+ @utf8=true
299
+ end
300
+
232
301
  def with_line(line)
233
- @line=line
302
+ @endline=line
234
303
  self
235
304
  end
236
305
 
306
+ def line; @endline end
307
+ def line= l; @endline=l end
308
+
237
309
  def initialize(type='"',ident='')
238
310
  super(ident)
239
311
  type=="'" and type='"'
@@ -241,7 +313,7 @@ class StringToken < Token
241
313
  assert @char[/^[\[{"`\/]$/] #"
242
314
  @elems=[ident.dup] #why .dup?
243
315
  @modifiers=nil
244
- @line=nil
316
+ @endline=nil
245
317
  end
246
318
 
247
319
  DQUOTE_ESCAPE_TABLE = [
@@ -291,6 +363,8 @@ class StringToken < Token
291
363
  result="%r"+result if RubyLexer::WHSPLF[result[1,1]]
292
364
  end
293
365
 
366
+ result<<"\n" if open.empty? and result[-1] != ?\n
367
+
294
368
  return result
295
369
  end
296
370
 
@@ -362,9 +436,9 @@ private
362
436
  UNESC_DELIMS={}
363
437
 
364
438
  #simpler transform, preserves original exactly
365
- def simple_transform(strfrag,starter,ender)
439
+ def simple_transform(strfrag,starter,ender) #appears to be unused
366
440
  assert('[{/'[@char])
367
- #strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='['
441
+ #strfrag.gsub!(/(\A|[^\\])(?:\\\\)*\#([{$@])/){$1+'\\#'+$2} unless @char=='[' #esc #{
368
442
  delimchars=Regexp.quote starter+ender
369
443
  delimchars+=Regexp.quote("#") unless @char=='[' #escape beginning of string iterpolations
370
444
 
@@ -390,7 +464,7 @@ private
390
464
  return strfrag
391
465
  end
392
466
 
393
- def transform(strfrag,starter,ender)
467
+ def transform(strfrag,starter,ender) #appears to be unused
394
468
  strfrag.gsub!("\\",'\\'*4)
395
469
  strfrag.gsub!(/#([{$@])/,'\\#\\1')
396
470
  strfrag.gsub!(Regexp.new("[\\"+starter+"\\"+ender+"]"),'\\\\\\&') unless @char=='?'
@@ -427,8 +501,9 @@ private
427
501
  end
428
502
 
429
503
  #-------------------------
430
- class RenderExactlyStringToken < StringToken
504
+ class RenderExactlyStringToken < StringToken
431
505
  alias transform simple_transform
506
+ #transform isn't called anymore, so there's no need for this hacky class
432
507
  end
433
508
 
434
509
  #-------------------------
@@ -471,9 +546,13 @@ class HerePlaceholderToken < WToken
471
546
 
472
547
  #def with_line(line) @string.line=line; self end
473
548
 
474
- def line; @string.line end
475
- def line=line; @string.line=line end
549
+ def line; @line || @string.line end
550
+ def line=line; @line=line end
476
551
 
552
+ def startline; @line end
553
+ alias endline startline
554
+ def startline=x; end
555
+ alias endline= startline=
477
556
  end
478
557
 
479
558
  #-------------------------
@@ -484,6 +563,13 @@ end
484
563
  #-------------------------
485
564
  class IgnoreToken < Token
486
565
  include StillIgnoreToken
566
+
567
+ def initialize(ident,*stuff)
568
+ @linecount=ident.count "\n"
569
+ super
570
+ end
571
+
572
+ attr :linecount
487
573
  end
488
574
 
489
575
  #-------------------------
@@ -511,6 +597,23 @@ class NoWsToken < ZwToken
511
597
  end
512
598
  end
513
599
 
600
+ #-------------------------
601
+ class ShebangToken < IgnoreToken
602
+ def initialize(text)
603
+ super text,0
604
+ end
605
+ end
606
+
607
+ #-------------------------
608
+ class EncodingDeclToken < IgnoreToken
609
+ def initialize(text,encoding,offset)
610
+ text||=''
611
+ super text,offset
612
+ @encoding=encoding
613
+ end
614
+ attr :encoding
615
+ end
616
+
514
617
  #-------------------------
515
618
  class ImplicitParamListStartToken < KeywordToken
516
619
  include StillIgnoreToken
@@ -579,6 +682,19 @@ class EscNlToken < IgnoreToken
579
682
  @filename=filename
580
683
  @linenum=linenum
581
684
  end
685
+
686
+ attr_accessor :filename,:linenum
687
+
688
+ def linecount; 1 end
689
+
690
+ def startline
691
+ @linenum-1
692
+ end
693
+ def endline
694
+ @linenum
695
+ end
696
+ def startline= bogus; end
697
+ alias endline= linenum=
582
698
  end
583
699
 
584
700
  #-------------------------
@@ -586,9 +702,10 @@ class EoiToken < Token
586
702
  attr :file
587
703
  alias :pos :offset
588
704
 
589
- def initialize(cause,file, offset=nil)
705
+ def initialize(cause,file, offset=nil,line=nil)
590
706
  super(cause,offset)
591
707
  @file=file
708
+ @endline=line
592
709
  end
593
710
  end
594
711
 
@@ -598,7 +715,7 @@ class HereBodyToken < IgnoreToken
598
715
  attr_accessor :open,:close
599
716
  def initialize(headtok,linecount)
600
717
  assert HerePlaceholderToken===headtok
601
- super(headtok.string,headtok.string.offset)
718
+ @ident,@offset=headtok.string,headtok.string.offset
602
719
  @headtok=headtok
603
720
  @linecount=linecount
604
721
  end
@@ -606,6 +723,14 @@ class HereBodyToken < IgnoreToken
606
723
  def line
607
724
  @ident.line
608
725
  end
726
+ alias endline line
727
+ def endline= line
728
+ @ident.line= line
729
+ end
730
+
731
+ def startline
732
+ line-@linecount+1
733
+ end
609
734
 
610
735
  def to_s
611
736
  @ident.to_s
@@ -629,15 +754,20 @@ class FileAndLineToken < IgnoreToken
629
754
  #def char; '#' end
630
755
 
631
756
  def to_s()
632
- ['#', @ident, ':', @line].to_s
757
+ %[##@ident:#@line]
633
758
  end
634
759
 
635
760
  def file() @ident end
636
761
  def subitem() @line end #needed?
762
+
763
+ def endline; @line end
764
+ def startline; @line end
765
+ alias endline= line=
766
+ def startline= bogus; end
637
767
  end
638
768
 
639
769
  #-------------------------
640
- class OutlinedHereBodyToken < HereBodyToken
770
+ class OutlinedHereBodyToken < HereBodyToken #appears to be unused
641
771
  def to_s
642
772
  assert HerePlaceholderToken===@headtok
643
773
  result=@headtok.string