rubylexer 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +90 -0
- data/Manifest.txt +54 -3
- data/README.txt +4 -7
- data/Rakefile +3 -2
- data/lib/rubylexer.rb +856 -323
- data/lib/rubylexer/0.7.0.rb +11 -2
- data/lib/rubylexer/0.7.1.rb +2 -0
- data/lib/rubylexer/charhandler.rb +4 -4
- data/lib/rubylexer/context.rb +86 -9
- data/lib/rubylexer/rulexer.rb +455 -101
- data/lib/rubylexer/token.rb +166 -43
- data/lib/rubylexer/tokenprinter.rb +16 -8
- data/lib/rubylexer/version.rb +1 -1
- data/rubylexer.vpj +98 -0
- data/test/code/all_the_gems.rb +33 -0
- data/test/code/all_the_raas.rb +226 -0
- data/test/code/all_the_rubies.rb +2 -0
- data/test/code/deletewarns.rb +19 -1
- data/test/code/dumptokens.rb +39 -8
- data/test/code/errscan +2 -0
- data/test/code/isolate_error.rb +72 -0
- data/test/code/lexloop +14 -0
- data/test/code/locatetest.rb +150 -8
- data/test/code/regression.rb +109 -0
- data/test/code/rubylexervsruby.rb +53 -15
- data/test/code/strgen.rb +138 -0
- data/test/code/tarball.rb +144 -0
- data/test/code/testcases.rb +11 -0
- data/test/code/tokentest.rb +115 -24
- data/test/data/__eof2.rb +1 -0
- data/test/data/__eof5.rb +2 -0
- data/test/data/__eof6.rb +2 -0
- data/test/data/cvtesc.rb +17 -0
- data/test/data/g.rb +6 -0
- data/test/data/hd0.rb +3 -0
- data/test/data/hdateof.rb +2 -0
- data/test/data/hdempty.rb +3 -0
- data/test/data/hdr.rb +9 -0
- data/test/data/hdr_dos.rb +13 -0
- data/test/data/hdr_dos2.rb +18 -0
- data/test/data/heart.rb +2 -0
- data/test/data/here_escnl.rb +25 -0
- data/test/data/here_escnl_dos.rb +20 -0
- data/test/data/here_squote.rb +3 -0
- data/test/data/heremonsters.rb +140 -0
- data/test/data/heremonsters.rb.broken +68 -0
- data/test/data/heremonsters.rb.broken.save +68 -0
- data/test/data/heremonsters_dos.rb +140 -0
- data/test/data/heremonsters_dos.rb.broken +68 -0
- data/test/data/illegal_oneliners.rb +1 -0
- data/test/data/illegal_stanzas.rb +0 -0
- data/test/data/make_ws_strdelim.rb +22 -0
- data/test/data/maven2_builer_test.rb +82 -0
- data/test/data/migration.rb +8944 -0
- data/test/data/modl.rb +6 -0
- data/test/data/modl_dos.rb +7 -0
- data/test/data/modl_fails.rb +10 -0
- data/test/data/multilinestring.rb +6 -0
- data/test/data/oneliners.rb +555 -0
- data/test/data/p-op.rb +2 -0
- data/test/data/p.rb +3 -1710
- data/test/data/s.rb +90 -21
- data/test/data/simple.rb +1 -0
- data/test/data/simple_dos.rb +1 -0
- data/test/data/stanzas.rb +1194 -0
- data/test/data/strdelim_crlf.rb +6 -0
- data/test/data/stuff.rb +6 -0
- data/test/data/stuff2.rb +5 -0
- data/test/data/stuff3.rb +6 -0
- data/test/data/stuff4.rb +6 -0
- data/test/data/tkweird.rb +20 -0
- data/test/data/unending_stuff.rb +5 -0
- data/test/data/whatnot.rb +8 -0
- data/test/data/ws_strdelim.rb +0 -0
- data/test/test.sh +239 -0
- data/testing.txt +39 -50
- metadata +110 -12
- data/test/code/dl_all_gems.rb +0 -43
- data/test/code/unpack_all_gems.rb +0 -15
- data/test/data/gemlist.txt +0 -280
data/test/data/__eof2.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
__END__
|
data/test/data/__eof5.rb
ADDED
data/test/data/__eof6.rb
ADDED
data/test/data/cvtesc.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
File.open(ARGV.first){|f|
|
2
|
+
deficit=0
|
3
|
+
while buf=f.read(1024+deficit)
|
4
|
+
#trim possibly incomplete sequences from end
|
5
|
+
if /\\[0-7]{0,2}\Z/===buf
|
6
|
+
deficit=$&.size
|
7
|
+
buf[-deficit..-1]=''
|
8
|
+
f.pos-=deficit
|
9
|
+
else
|
10
|
+
deficit=0
|
11
|
+
end
|
12
|
+
|
13
|
+
$stdout.write buf.gsub(/\\([0-7]{1,3})/){
|
14
|
+
$1.oct.&(0xFF).chr
|
15
|
+
}
|
16
|
+
end
|
17
|
+
}
|
data/test/data/g.rb
CHANGED
data/test/data/hd0.rb
ADDED
data/test/data/hdr.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
00000000 63 6f 6e 74 65 6e 74 73 20 3c 3c 20 3c 3c 48 45 |contents << <<HE|
|
2
|
+
00000010 41 44 45 52 0d 0d 0a 63 6c 61 73 73 20 23 7b 66 |ADER...class #{f|
|
3
|
+
00000020 69 6c 65 6e 61 6d 65 2e 63 61 70 69 74 61 6c 69 |ilename.capitali|
|
4
|
+
00000030 7a 65 7d 54 65 73 74 43 61 73 65 20 3c 20 4d 61 |ze}TestCase < Ma|
|
5
|
+
00000040 74 68 33 64 3a 3a 54 65 73 74 43 61 73 65 0d 0d |th3d::TestCase..|
|
6
|
+
00000050 0a 0d 0d 0a 20 20 20 20 20 20 20 20 64 65 66 20 |.... def |
|
7
|
+
00000060 73 65 74 5f 75 70 0d 0d 0a 20 20 20 20 20 20 20 |set_up... |
|
8
|
+
00000070 20 20 20 20 20 20 20 20 20 40 23 7b 66 69 6c 65 | @#{file|
|
9
|
+
00000080 6e 61 6d 65 2e 64 6f 77 6e 63 61 73 65 7d 73 20 |name.downcase}s |
|
10
|
+
00000090 3d 20 5b 5d 0d 0d 0a 20 20 20 20 20 20 20 20 20 |= []... |
|
11
|
+
000000a0 20 20 20 20 20 20 20 40 23 7b 66 69 6c 65 6e 61 | @#{filena|
|
12
|
+
000000b0 6d 65 2e 64 6f 77 6e 63 61 73 65 7d 73 2e 63 6c |me.downcase}s.cl|
|
13
|
+
000000c0 65 61 72 0d 0d 0a 20 20 20 20 20 20 20 20 65 6e |ear... en|
|
14
|
+
000000d0 64 0d 0d 0a 0d 0d 0a 48 45 41 44 45 52 0d 0d 0a |d......HEADER...|
|
15
|
+
000000e0 22 22 20 3c 3c 20 3c 3c 53 74 75 70 69 64 0d 0d |"" << <<Stupid..|
|
16
|
+
000000f0 0a 65 6e 64 0d 0d 0a 53 74 75 70 69 64 0d 0d 0a |.end...Stupid...|
|
17
|
+
00000100 0d 0d 0a |...|
|
18
|
+
00000103
|
data/test/data/heart.rb
CHANGED
@@ -0,0 +1,25 @@
|
|
1
|
+
p(<<stuff+'foobar')\
|
2
|
+
+"more stuff"
|
3
|
+
some stuff
|
4
|
+
stuff
|
5
|
+
|
6
|
+
p <<stuff+'foobar'\
|
7
|
+
+"more stuff"
|
8
|
+
some stuff
|
9
|
+
stuff
|
10
|
+
|
11
|
+
p <<stuff+'foobar'+
|
12
|
+
some stuff
|
13
|
+
stuff
|
14
|
+
"more stuff"
|
15
|
+
|
16
|
+
p <<stuff+'foobar'+<<baggage+"baz"+<<impedimentia+
|
17
|
+
some stuff
|
18
|
+
stuff
|
19
|
+
blah blah
|
20
|
+
baggage
|
21
|
+
hic haec hoc
|
22
|
+
impedimentia
|
23
|
+
"more stuff"
|
24
|
+
|
25
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
p <<stuff+'foobar'\
|
2
|
+
+"more stuff"
|
3
|
+
some stuff
|
4
|
+
stuff
|
5
|
+
|
6
|
+
p <<stuff+'foobar'+
|
7
|
+
some stuff
|
8
|
+
stuff
|
9
|
+
"more stuff"
|
10
|
+
|
11
|
+
p <<stuff+'foobar'+<<baggage+"baz"+<<impedimentia+
|
12
|
+
some stuff
|
13
|
+
stuff
|
14
|
+
blah blah
|
15
|
+
baggage
|
16
|
+
hic haec hoc
|
17
|
+
impedimentia
|
18
|
+
"more stuff"
|
19
|
+
|
20
|
+
|
@@ -0,0 +1,140 @@
|
|
1
|
+
<<END0+<<END1+<<END2+"kdsjfasdjf"
|
2
|
+
#{m0}
|
3
|
+
END0
|
4
|
+
#{m1}
|
5
|
+
END1
|
6
|
+
#{m2}
|
7
|
+
END2
|
8
|
+
|
9
|
+
foo a=<<a,b=<<b,c=<<c
|
10
|
+
jfksdkjf
|
11
|
+
dkljjkf
|
12
|
+
a
|
13
|
+
kdljfjkdg
|
14
|
+
dfglkdfkgjdf
|
15
|
+
dkf
|
16
|
+
b
|
17
|
+
lkdffdjksadhf
|
18
|
+
sdflkdjgsfdkjgsdg
|
19
|
+
dsfg;lkdflisgffd
|
20
|
+
g
|
21
|
+
c
|
22
|
+
|
23
|
+
foo(a=<<a,b=<<b,c=<<c)
|
24
|
+
jfksdkjf
|
25
|
+
dkljjkf
|
26
|
+
a
|
27
|
+
kdljfjkdg
|
28
|
+
dfglkdfkgjdf
|
29
|
+
dkf
|
30
|
+
b
|
31
|
+
lkdffdjksadhf
|
32
|
+
sdflkdjgsfdkjgsdg
|
33
|
+
dsfg;lkdflisgffd
|
34
|
+
g
|
35
|
+
c
|
36
|
+
|
37
|
+
def foo(a=<<a,b=<<b,c=<<c)
|
38
|
+
jfksdkjf
|
39
|
+
dkljjkf
|
40
|
+
a
|
41
|
+
kdljfjkdg
|
42
|
+
dfglkdfkgjdf
|
43
|
+
dkf
|
44
|
+
b
|
45
|
+
lkdffdjksadhf
|
46
|
+
sdflkdjgsfdkjgsdg
|
47
|
+
dsfg;lkdflisgffd
|
48
|
+
g
|
49
|
+
c
|
50
|
+
|
51
|
+
a+b+c
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
p <<stuff+'foobar'.tr('j-l','d-f')\
|
56
|
+
+"more stuff"
|
57
|
+
12345678
|
58
|
+
the quick brown fox jumped over the lazy dog
|
59
|
+
stuff
|
60
|
+
|
61
|
+
p <<simple1
|
62
|
+
foo
|
63
|
+
bar
|
64
|
+
baz
|
65
|
+
simple1
|
66
|
+
p(<<simple2)
|
67
|
+
foo2
|
68
|
+
bar
|
69
|
+
baz
|
70
|
+
simple2
|
71
|
+
|
72
|
+
p <<oof+"gfert"
|
73
|
+
#{gleeble #breakpoint
|
74
|
+
}
|
75
|
+
oof
|
76
|
+
|
77
|
+
funcnames.collect{|fn| <<-endeval; hn }.to_s(fn,hn,gn=1)
|
78
|
+
#{fn} #{gn} #{hn=2}
|
79
|
+
#$fn #@gn #@@hn=2
|
80
|
+
endeval
|
81
|
+
p fn,gn,hn
|
82
|
+
|
83
|
+
p <<foo+'crazy'+<<bar+'nutty'
|
84
|
+
sdsdf
|
85
|
+
foo
|
86
|
+
sdfsdfgsdfsdf
|
87
|
+
bar
|
88
|
+
|
89
|
+
p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'
|
90
|
+
sdsdf
|
91
|
+
foo
|
92
|
+
sdfsdfgsdfsdf
|
93
|
+
bar
|
94
|
+
uityeiorwpieo
|
95
|
+
baz
|
96
|
+
|
97
|
+
p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'
|
98
|
+
sdsdf
|
99
|
+
foo
|
100
|
+
sdfsdfgsdfsdf
|
101
|
+
bar
|
102
|
+
uityeiorwpieo
|
103
|
+
baz
|
104
|
+
CVBNXVBdfg
|
105
|
+
quux
|
106
|
+
|
107
|
+
p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'+
|
108
|
+
sdsdf
|
109
|
+
foo
|
110
|
+
sdfsdfgsdfsdf
|
111
|
+
bar
|
112
|
+
uityeiorwpieo
|
113
|
+
baz
|
114
|
+
CVBNXVBdfg
|
115
|
+
quux
|
116
|
+
"5645647585735"
|
117
|
+
|
118
|
+
p <<foo+
|
119
|
+
bar
|
120
|
+
baz
|
121
|
+
foo
|
122
|
+
"quux"
|
123
|
+
|
124
|
+
|
125
|
+
p <<f+<<f
|
126
|
+
s435fg
|
127
|
+
f
|
128
|
+
2345gdf
|
129
|
+
f
|
130
|
+
|
131
|
+
p <<p
|
132
|
+
q\
|
133
|
+
p
|
134
|
+
p
|
135
|
+
|
136
|
+
p <<''
|
137
|
+
q\
|
138
|
+
|
139
|
+
p
|
140
|
+
|
@@ -0,0 +1,68 @@
|
|
1
|
+
#offsets in these cases are incorrect... maybe never
|
2
|
+
|
3
|
+
p <<-foo+'123
|
4
|
+
abcd
|
5
|
+
foo
|
6
|
+
456'+"8910"
|
7
|
+
p heh
|
8
|
+
|
9
|
+
p <<-foo+"123#{2.718281828}
|
10
|
+
abcd
|
11
|
+
foo
|
12
|
+
456#{3.14159265}"+"8910"
|
13
|
+
p hurhurhurhurhur!
|
14
|
+
|
15
|
+
p <<-foo+"123#{2.718281828}
|
16
|
+
abcd
|
17
|
+
foo
|
18
|
+
456#{3.14159265}"
|
19
|
+
p hurhurhurhurhur!
|
20
|
+
|
21
|
+
|
22
|
+
p <<here+'123
|
23
|
+
456
|
24
|
+
there
|
25
|
+
here
|
26
|
+
789'
|
27
|
+
p snort!
|
28
|
+
|
29
|
+
p <<here+'123\
|
30
|
+
456
|
31
|
+
there
|
32
|
+
here
|
33
|
+
789'
|
34
|
+
p snort!
|
35
|
+
|
36
|
+
p <<here+'123\
|
37
|
+
456
|
38
|
+
there
|
39
|
+
here
|
40
|
+
789'
|
41
|
+
|
42
|
+
p <<-foo+'123
|
43
|
+
abcd
|
44
|
+
foo
|
45
|
+
456'
|
46
|
+
p ha-ha
|
47
|
+
|
48
|
+
p <<-foo+'123tyurui
|
49
|
+
abcdCWECWWE
|
50
|
+
foo
|
51
|
+
456rtjmnj'
|
52
|
+
p ha-ha
|
53
|
+
|
54
|
+
|
55
|
+
p "#{<<foobar1.each('|'){|s| '\nthort: '+s}
|
56
|
+
jbvd|g4543ghb|!@G$dfsd|fafr|e
|
57
|
+
|s4e5rrwware|BBBBB|*&^(*&^>"PMK:njs;d|
|
58
|
+
|
59
|
+
foobar1
|
60
|
+
}"
|
61
|
+
|
62
|
+
|
63
|
+
p <<bazquux+"#{
|
64
|
+
dfgnb
|
65
|
+
t 67sevrgvase
|
66
|
+
234vjvj7ui
|
67
|
+
bazquux
|
68
|
+
}foobar"
|
@@ -0,0 +1,68 @@
|
|
1
|
+
#offsets in these cases are incorrect... maybe never
|
2
|
+
|
3
|
+
p <<-foo+'123
|
4
|
+
abcd
|
5
|
+
foo
|
6
|
+
456'+"8910"
|
7
|
+
p heh
|
8
|
+
|
9
|
+
p <<-foo+"123#{2.718281828}
|
10
|
+
abcd
|
11
|
+
foo
|
12
|
+
456#{3.14159265}"+"8910"
|
13
|
+
p hurhurhurhurhur!
|
14
|
+
|
15
|
+
p <<-foo+"123#{2.718281828}
|
16
|
+
abcd
|
17
|
+
foo
|
18
|
+
456#{3.14159265}"
|
19
|
+
p hurhurhurhurhur!
|
20
|
+
|
21
|
+
|
22
|
+
p <<here+'123
|
23
|
+
456
|
24
|
+
there
|
25
|
+
here
|
26
|
+
789'
|
27
|
+
p snort!
|
28
|
+
|
29
|
+
p <<here+'123\
|
30
|
+
456
|
31
|
+
there
|
32
|
+
here
|
33
|
+
789'
|
34
|
+
p snort!
|
35
|
+
|
36
|
+
p <<here+'123\
|
37
|
+
456
|
38
|
+
there
|
39
|
+
here
|
40
|
+
789'
|
41
|
+
|
42
|
+
p <<-foo+'123
|
43
|
+
abcd
|
44
|
+
foo
|
45
|
+
456'
|
46
|
+
p ha-ha
|
47
|
+
|
48
|
+
p <<-foo+'123tyurui
|
49
|
+
abcdCWECWWE
|
50
|
+
foo
|
51
|
+
456rtjmnj'
|
52
|
+
p ha-ha
|
53
|
+
|
54
|
+
|
55
|
+
p "#{<<foobar3}"
|
56
|
+
bim
|
57
|
+
baz
|
58
|
+
bof
|
59
|
+
foobar3
|
60
|
+
|
61
|
+
p <<one ; p "#{<<two}"
|
62
|
+
1111111111111111
|
63
|
+
fdsgdfgdsfffff
|
64
|
+
one
|
65
|
+
2222222222222222
|
66
|
+
sdfasdfasdfads
|
67
|
+
two
|
68
|
+
|
@@ -0,0 +1,140 @@
|
|
1
|
+
<<END0+<<END1+<<END2+"kdsjfasdjf"
|
2
|
+
#{m0}
|
3
|
+
END0
|
4
|
+
#{m1}
|
5
|
+
END1
|
6
|
+
#{m2}
|
7
|
+
END2
|
8
|
+
|
9
|
+
foo a=<<a,b=<<b,c=<<c
|
10
|
+
jfksdkjf
|
11
|
+
dkljjkf
|
12
|
+
a
|
13
|
+
kdljfjkdg
|
14
|
+
dfglkdfkgjdf
|
15
|
+
dkf
|
16
|
+
b
|
17
|
+
lkdffdjksadhf
|
18
|
+
sdflkdjgsfdkjgsdg
|
19
|
+
dsfg;lkdflisgffd
|
20
|
+
g
|
21
|
+
c
|
22
|
+
|
23
|
+
foo(a=<<a,b=<<b,c=<<c)
|
24
|
+
jfksdkjf
|
25
|
+
dkljjkf
|
26
|
+
a
|
27
|
+
kdljfjkdg
|
28
|
+
dfglkdfkgjdf
|
29
|
+
dkf
|
30
|
+
b
|
31
|
+
lkdffdjksadhf
|
32
|
+
sdflkdjgsfdkjgsdg
|
33
|
+
dsfg;lkdflisgffd
|
34
|
+
g
|
35
|
+
c
|
36
|
+
|
37
|
+
def foo(a=<<a,b=<<b,c=<<c)
|
38
|
+
jfksdkjf
|
39
|
+
dkljjkf
|
40
|
+
a
|
41
|
+
kdljfjkdg
|
42
|
+
dfglkdfkgjdf
|
43
|
+
dkf
|
44
|
+
b
|
45
|
+
lkdffdjksadhf
|
46
|
+
sdflkdjgsfdkjgsdg
|
47
|
+
dsfg;lkdflisgffd
|
48
|
+
g
|
49
|
+
c
|
50
|
+
|
51
|
+
a+b+c
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
p <<stuff+'foobar'.tr('j-l','d-f')\
|
56
|
+
+"more stuff"
|
57
|
+
12345678
|
58
|
+
the quick brown fox jumped over the lazy dog
|
59
|
+
stuff
|
60
|
+
|
61
|
+
p <<simple1
|
62
|
+
foo
|
63
|
+
bar
|
64
|
+
baz
|
65
|
+
simple1
|
66
|
+
p(<<simple2)
|
67
|
+
foo2
|
68
|
+
bar
|
69
|
+
baz
|
70
|
+
simple2
|
71
|
+
|
72
|
+
p <<oof+"gfert"
|
73
|
+
#{gleeble #breakpoint
|
74
|
+
}
|
75
|
+
oof
|
76
|
+
|
77
|
+
funcnames.collect{|fn| <<-endeval; hn }.to_s(fn,hn,gn=1)
|
78
|
+
#{fn} #{gn} #{hn=2}
|
79
|
+
#$fn #@gn #@@hn=2
|
80
|
+
endeval
|
81
|
+
p fn,gn,hn
|
82
|
+
|
83
|
+
p <<foo+'crazy'+<<bar+'nutty'
|
84
|
+
sdsdf
|
85
|
+
foo
|
86
|
+
sdfsdfgsdfsdf
|
87
|
+
bar
|
88
|
+
|
89
|
+
p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'
|
90
|
+
sdsdf
|
91
|
+
foo
|
92
|
+
sdfsdfgsdfsdf
|
93
|
+
bar
|
94
|
+
uityeiorwpieo
|
95
|
+
baz
|
96
|
+
|
97
|
+
p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'
|
98
|
+
sdsdf
|
99
|
+
foo
|
100
|
+
sdfsdfgsdfsdf
|
101
|
+
bar
|
102
|
+
uityeiorwpieo
|
103
|
+
baz
|
104
|
+
CVBNXVBdfg
|
105
|
+
quux
|
106
|
+
|
107
|
+
p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'+
|
108
|
+
sdsdf
|
109
|
+
foo
|
110
|
+
sdfsdfgsdfsdf
|
111
|
+
bar
|
112
|
+
uityeiorwpieo
|
113
|
+
baz
|
114
|
+
CVBNXVBdfg
|
115
|
+
quux
|
116
|
+
"5645647585735"
|
117
|
+
|
118
|
+
p <<foo+
|
119
|
+
bar
|
120
|
+
baz
|
121
|
+
foo
|
122
|
+
"quux"
|
123
|
+
|
124
|
+
|
125
|
+
p <<f+<<f
|
126
|
+
s435fg
|
127
|
+
f
|
128
|
+
2345gdf
|
129
|
+
f
|
130
|
+
|
131
|
+
p <<p
|
132
|
+
q\
|
133
|
+
p
|
134
|
+
p
|
135
|
+
|
136
|
+
p <<''
|
137
|
+
q\
|
138
|
+
|
139
|
+
p
|
140
|
+
|