rubylexer 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/History.txt +90 -0
  2. data/Manifest.txt +54 -3
  3. data/README.txt +4 -7
  4. data/Rakefile +3 -2
  5. data/lib/rubylexer.rb +856 -323
  6. data/lib/rubylexer/0.7.0.rb +11 -2
  7. data/lib/rubylexer/0.7.1.rb +2 -0
  8. data/lib/rubylexer/charhandler.rb +4 -4
  9. data/lib/rubylexer/context.rb +86 -9
  10. data/lib/rubylexer/rulexer.rb +455 -101
  11. data/lib/rubylexer/token.rb +166 -43
  12. data/lib/rubylexer/tokenprinter.rb +16 -8
  13. data/lib/rubylexer/version.rb +1 -1
  14. data/rubylexer.vpj +98 -0
  15. data/test/code/all_the_gems.rb +33 -0
  16. data/test/code/all_the_raas.rb +226 -0
  17. data/test/code/all_the_rubies.rb +2 -0
  18. data/test/code/deletewarns.rb +19 -1
  19. data/test/code/dumptokens.rb +39 -8
  20. data/test/code/errscan +2 -0
  21. data/test/code/isolate_error.rb +72 -0
  22. data/test/code/lexloop +14 -0
  23. data/test/code/locatetest.rb +150 -8
  24. data/test/code/regression.rb +109 -0
  25. data/test/code/rubylexervsruby.rb +53 -15
  26. data/test/code/strgen.rb +138 -0
  27. data/test/code/tarball.rb +144 -0
  28. data/test/code/testcases.rb +11 -0
  29. data/test/code/tokentest.rb +115 -24
  30. data/test/data/__eof2.rb +1 -0
  31. data/test/data/__eof5.rb +2 -0
  32. data/test/data/__eof6.rb +2 -0
  33. data/test/data/cvtesc.rb +17 -0
  34. data/test/data/g.rb +6 -0
  35. data/test/data/hd0.rb +3 -0
  36. data/test/data/hdateof.rb +2 -0
  37. data/test/data/hdempty.rb +3 -0
  38. data/test/data/hdr.rb +9 -0
  39. data/test/data/hdr_dos.rb +13 -0
  40. data/test/data/hdr_dos2.rb +18 -0
  41. data/test/data/heart.rb +2 -0
  42. data/test/data/here_escnl.rb +25 -0
  43. data/test/data/here_escnl_dos.rb +20 -0
  44. data/test/data/here_squote.rb +3 -0
  45. data/test/data/heremonsters.rb +140 -0
  46. data/test/data/heremonsters.rb.broken +68 -0
  47. data/test/data/heremonsters.rb.broken.save +68 -0
  48. data/test/data/heremonsters_dos.rb +140 -0
  49. data/test/data/heremonsters_dos.rb.broken +68 -0
  50. data/test/data/illegal_oneliners.rb +1 -0
  51. data/test/data/illegal_stanzas.rb +0 -0
  52. data/test/data/make_ws_strdelim.rb +22 -0
  53. data/test/data/maven2_builer_test.rb +82 -0
  54. data/test/data/migration.rb +8944 -0
  55. data/test/data/modl.rb +6 -0
  56. data/test/data/modl_dos.rb +7 -0
  57. data/test/data/modl_fails.rb +10 -0
  58. data/test/data/multilinestring.rb +6 -0
  59. data/test/data/oneliners.rb +555 -0
  60. data/test/data/p-op.rb +2 -0
  61. data/test/data/p.rb +3 -1710
  62. data/test/data/s.rb +90 -21
  63. data/test/data/simple.rb +1 -0
  64. data/test/data/simple_dos.rb +1 -0
  65. data/test/data/stanzas.rb +1194 -0
  66. data/test/data/strdelim_crlf.rb +6 -0
  67. data/test/data/stuff.rb +6 -0
  68. data/test/data/stuff2.rb +5 -0
  69. data/test/data/stuff3.rb +6 -0
  70. data/test/data/stuff4.rb +6 -0
  71. data/test/data/tkweird.rb +20 -0
  72. data/test/data/unending_stuff.rb +5 -0
  73. data/test/data/whatnot.rb +8 -0
  74. data/test/data/ws_strdelim.rb +0 -0
  75. data/test/test.sh +239 -0
  76. data/testing.txt +39 -50
  77. metadata +110 -12
  78. data/test/code/dl_all_gems.rb +0 -43
  79. data/test/code/unpack_all_gems.rb +0 -15
  80. data/test/data/gemlist.txt +0 -280
data/test/data/__eof2.rb ADDED
@@ -0,0 +1 @@
1
+ __END__
data/test/data/__eof5.rb ADDED
@@ -0,0 +1,2 @@
1
+ BEGIN{44}; \
2
+ __END__
data/test/data/__eof6.rb ADDED
@@ -0,0 +1,2 @@
1
+ BEGIN{44}; \
2
+ __END__
data/test/data/cvtesc.rb ADDED
@@ -0,0 +1,17 @@
1
+ File.open(ARGV.first){|f|
2
+ deficit=0
3
+ while buf=f.read(1024+deficit)
4
+ #trim possibly incomplete sequences from end
5
+ if /\\[0-7]{0,2}\Z/===buf
6
+ deficit=$&.size
7
+ buf[-deficit..-1]=''
8
+ f.pos-=deficit
9
+ else
10
+ deficit=0
11
+ end
12
+
13
+ $stdout.write buf.gsub(/\\([0-7]{1,3})/){
14
+ $1.oct.&(0xFF).chr
15
+ }
16
+ end
17
+ }
data/test/data/g.rb CHANGED
@@ -13,3 +13,9 @@ p(gg :z)
13
13
  # g :w #error
14
14
 
15
15
 
16
+ begin
17
+ a
18
+ rescue
19
+ b;c
20
+ end
21
+
data/test/data/hd0.rb ADDED
@@ -0,0 +1,3 @@
1
+ <<p
2
+ p
3
+
data/test/data/hdateof.rb ADDED
@@ -0,0 +1,2 @@
1
+ <<E
2
+ E
data/test/data/hdempty.rb ADDED
@@ -0,0 +1,3 @@
1
+ p <<f
2
+ f
3
+
data/test/data/hdr.rb ADDED
@@ -0,0 +1,9 @@
1
+ <<HEADER
2
+ class #{filename.capitalize}TestCase
3
+ def set_up
4
+ @#{filename.downcase}s
5
+ @#{filename.downcase}s
6
+ end
7
+ end
8
+ HEADER
9
+
data/test/data/hdr_dos.rb ADDED
@@ -0,0 +1,13 @@
1
+ contents << <<HEADER
2
+ class #{filename.capitalize}TestCase < Math3d::TestCase
3
+
4
+ def set_up
5
+ @#{filename.downcase}s = []
6
+ @#{filename.downcase}s.clear
7
+ end
8
+
9
+ HEADER
10
+ "" << <<Stupid
11
+ end
12
+ Stupid
13
+
data/test/data/hdr_dos2.rb ADDED
@@ -0,0 +1,18 @@
1
+ 00000000 63 6f 6e 74 65 6e 74 73 20 3c 3c 20 3c 3c 48 45 |contents << <<HE|
2
+ 00000010 41 44 45 52 0d 0d 0a 63 6c 61 73 73 20 23 7b 66 |ADER...class #{f|
3
+ 00000020 69 6c 65 6e 61 6d 65 2e 63 61 70 69 74 61 6c 69 |ilename.capitali|
4
+ 00000030 7a 65 7d 54 65 73 74 43 61 73 65 20 3c 20 4d 61 |ze}TestCase < Ma|
5
+ 00000040 74 68 33 64 3a 3a 54 65 73 74 43 61 73 65 0d 0d |th3d::TestCase..|
6
+ 00000050 0a 0d 0d 0a 20 20 20 20 20 20 20 20 64 65 66 20 |.... def |
7
+ 00000060 73 65 74 5f 75 70 0d 0d 0a 20 20 20 20 20 20 20 |set_up... |
8
+ 00000070 20 20 20 20 20 20 20 20 20 40 23 7b 66 69 6c 65 | @#{file|
9
+ 00000080 6e 61 6d 65 2e 64 6f 77 6e 63 61 73 65 7d 73 20 |name.downcase}s |
10
+ 00000090 3d 20 5b 5d 0d 0d 0a 20 20 20 20 20 20 20 20 20 |= []... |
11
+ 000000a0 20 20 20 20 20 20 20 40 23 7b 66 69 6c 65 6e 61 | @#{filena|
12
+ 000000b0 6d 65 2e 64 6f 77 6e 63 61 73 65 7d 73 2e 63 6c |me.downcase}s.cl|
13
+ 000000c0 65 61 72 0d 0d 0a 20 20 20 20 20 20 20 20 65 6e |ear... en|
14
+ 000000d0 64 0d 0d 0a 0d 0d 0a 48 45 41 44 45 52 0d 0d 0a |d......HEADER...|
15
+ 000000e0 22 22 20 3c 3c 20 3c 3c 53 74 75 70 69 64 0d 0d |"" << <<Stupid..|
16
+ 000000f0 0a 65 6e 64 0d 0d 0a 53 74 75 70 69 64 0d 0d 0a |.end...Stupid...|
17
+ 00000100 0d 0d 0a |...|
18
+ 00000103
data/test/data/heart.rb CHANGED
@@ -4,4 +4,6 @@ def ♥(name)
4
4
  end
5
5
 
6
6
  ♥ ARGV.shift
7
+ #p ?♥ #not supported yet
8
+ p :♥
7
9
  #♥
data/test/data/here_escnl.rb ADDED
@@ -0,0 +1,25 @@
1
+ p(<<stuff+'foobar')\
2
+ +"more stuff"
3
+ some stuff
4
+ stuff
5
+
6
+ p <<stuff+'foobar'\
7
+ +"more stuff"
8
+ some stuff
9
+ stuff
10
+
11
+ p <<stuff+'foobar'+
12
+ some stuff
13
+ stuff
14
+ "more stuff"
15
+
16
+ p <<stuff+'foobar'+<<baggage+"baz"+<<impedimentia+
17
+ some stuff
18
+ stuff
19
+ blah blah
20
+ baggage
21
+ hic haec hoc
22
+ impedimentia
23
+ "more stuff"
24
+
25
+
data/test/data/here_escnl_dos.rb ADDED
@@ -0,0 +1,20 @@
1
+ p <<stuff+'foobar'\
2
+ +"more stuff"
3
+ some stuff
4
+ stuff
5
+
6
+ p <<stuff+'foobar'+
7
+ some stuff
8
+ stuff
9
+ "more stuff"
10
+
11
+ p <<stuff+'foobar'+<<baggage+"baz"+<<impedimentia+
12
+ some stuff
13
+ stuff
14
+ blah blah
15
+ baggage
16
+ hic haec hoc
17
+ impedimentia
18
+ "more stuff"
19
+
20
+
data/test/data/here_squote.rb ADDED
@@ -0,0 +1,3 @@
1
+ <<'E'
2
+ 1
3
+ E
data/test/data/heremonsters.rb ADDED
@@ -0,0 +1,140 @@
1
+ <<END0+<<END1+<<END2+"kdsjfasdjf"
2
+ #{m0}
3
+ END0
4
+ #{m1}
5
+ END1
6
+ #{m2}
7
+ END2
8
+
9
+ foo a=<<a,b=<<b,c=<<c
10
+ jfksdkjf
11
+ dkljjkf
12
+ a
13
+ kdljfjkdg
14
+ dfglkdfkgjdf
15
+ dkf
16
+ b
17
+ lkdffdjksadhf
18
+ sdflkdjgsfdkjgsdg
19
+ dsfg;lkdflisgffd
20
+ g
21
+ c
22
+
23
+ foo(a=<<a,b=<<b,c=<<c)
24
+ jfksdkjf
25
+ dkljjkf
26
+ a
27
+ kdljfjkdg
28
+ dfglkdfkgjdf
29
+ dkf
30
+ b
31
+ lkdffdjksadhf
32
+ sdflkdjgsfdkjgsdg
33
+ dsfg;lkdflisgffd
34
+ g
35
+ c
36
+
37
+ def foo(a=<<a,b=<<b,c=<<c)
38
+ jfksdkjf
39
+ dkljjkf
40
+ a
41
+ kdljfjkdg
42
+ dfglkdfkgjdf
43
+ dkf
44
+ b
45
+ lkdffdjksadhf
46
+ sdflkdjgsfdkjgsdg
47
+ dsfg;lkdflisgffd
48
+ g
49
+ c
50
+
51
+ a+b+c
52
+
53
+ end
54
+
55
+ p <<stuff+'foobar'.tr('j-l','d-f')\
56
+ +"more stuff"
57
+ 12345678
58
+ the quick brown fox jumped over the lazy dog
59
+ stuff
60
+
61
+ p <<simple1
62
+ foo
63
+ bar
64
+ baz
65
+ simple1
66
+ p(<<simple2)
67
+ foo2
68
+ bar
69
+ baz
70
+ simple2
71
+
72
+ p <<oof+"gfert"
73
+ #{gleeble #breakpoint
74
+ }
75
+ oof
76
+
77
+ funcnames.collect{|fn| <<-endeval; hn }.to_s(fn,hn,gn=1)
78
+ #{fn} #{gn} #{hn=2}
79
+ #$fn #@gn #@@hn=2
80
+ endeval
81
+ p fn,gn,hn
82
+
83
+ p <<foo+'crazy'+<<bar+'nutty'
84
+ sdsdf
85
+ foo
86
+ sdfsdfgsdfsdf
87
+ bar
88
+
89
+ p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'
90
+ sdsdf
91
+ foo
92
+ sdfsdfgsdfsdf
93
+ bar
94
+ uityeiorwpieo
95
+ baz
96
+
97
+ p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'
98
+ sdsdf
99
+ foo
100
+ sdfsdfgsdfsdf
101
+ bar
102
+ uityeiorwpieo
103
+ baz
104
+ CVBNXVBdfg
105
+ quux
106
+
107
+ p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'+
108
+ sdsdf
109
+ foo
110
+ sdfsdfgsdfsdf
111
+ bar
112
+ uityeiorwpieo
113
+ baz
114
+ CVBNXVBdfg
115
+ quux
116
+ "5645647585735"
117
+
118
+ p <<foo+
119
+ bar
120
+ baz
121
+ foo
122
+ "quux"
123
+
124
+
125
+ p <<f+<<f
126
+ s435fg
127
+ f
128
+ 2345gdf
129
+ f
130
+
131
+ p <<p
132
+ q\
133
+ p
134
+ p
135
+
136
+ p <<''
137
+ q\
138
+
139
+ p
140
+
data/test/data/heremonsters.rb.broken ADDED
@@ -0,0 +1,68 @@
1
+ #offsets in these cases are incorrect... maybe never
2
+
3
+ p <<-foo+'123
4
+ abcd
5
+ foo
6
+ 456'+"8910"
7
+ p heh
8
+
9
+ p <<-foo+"123#{2.718281828}
10
+ abcd
11
+ foo
12
+ 456#{3.14159265}"+"8910"
13
+ p hurhurhurhurhur!
14
+
15
+ p <<-foo+"123#{2.718281828}
16
+ abcd
17
+ foo
18
+ 456#{3.14159265}"
19
+ p hurhurhurhurhur!
20
+
21
+
22
+ p <<here+'123
23
+ 456
24
+ there
25
+ here
26
+ 789'
27
+ p snort!
28
+
29
+ p <<here+'123\
30
+ 456
31
+ there
32
+ here
33
+ 789'
34
+ p snort!
35
+
36
+ p <<here+'123\
37
+ 456
38
+ there
39
+ here
40
+ 789'
41
+
42
+ p <<-foo+'123
43
+ abcd
44
+ foo
45
+ 456'
46
+ p ha-ha
47
+
48
+ p <<-foo+'123tyurui
49
+ abcdCWECWWE
50
+ foo
51
+ 456rtjmnj'
52
+ p ha-ha
53
+
54
+
55
+ p "#{<<foobar1.each('|'){|s| '\nthort: '+s}
56
+ jbvd|g4543ghb|!@G$dfsd|fafr|e
57
+ |s4e5rrwware|BBBBB|*&^(*&^>"PMK:njs;d|
58
+
59
+ foobar1
60
+ }"
61
+
62
+
63
+ p <<bazquux+"#{
64
+ dfgnb
65
+ t 67sevrgvase
66
+ 234vjvj7ui
67
+ bazquux
68
+ }foobar"
data/test/data/heremonsters.rb.broken.save ADDED
@@ -0,0 +1,68 @@
1
+ #offsets in these cases are incorrect... maybe never
2
+
3
+ p <<-foo+'123
4
+ abcd
5
+ foo
6
+ 456'+"8910"
7
+ p heh
8
+
9
+ p <<-foo+"123#{2.718281828}
10
+ abcd
11
+ foo
12
+ 456#{3.14159265}"+"8910"
13
+ p hurhurhurhurhur!
14
+
15
+ p <<-foo+"123#{2.718281828}
16
+ abcd
17
+ foo
18
+ 456#{3.14159265}"
19
+ p hurhurhurhurhur!
20
+
21
+
22
+ p <<here+'123
23
+ 456
24
+ there
25
+ here
26
+ 789'
27
+ p snort!
28
+
29
+ p <<here+'123\
30
+ 456
31
+ there
32
+ here
33
+ 789'
34
+ p snort!
35
+
36
+ p <<here+'123\
37
+ 456
38
+ there
39
+ here
40
+ 789'
41
+
42
+ p <<-foo+'123
43
+ abcd
44
+ foo
45
+ 456'
46
+ p ha-ha
47
+
48
+ p <<-foo+'123tyurui
49
+ abcdCWECWWE
50
+ foo
51
+ 456rtjmnj'
52
+ p ha-ha
53
+
54
+
55
+ p "#{<<foobar3}"
56
+ bim
57
+ baz
58
+ bof
59
+ foobar3
60
+
61
+ p <<one ; p "#{<<two}"
62
+ 1111111111111111
63
+ fdsgdfgdsfffff
64
+ one
65
+ 2222222222222222
66
+ sdfasdfasdfads
67
+ two
68
+
data/test/data/heremonsters_dos.rb ADDED
@@ -0,0 +1,140 @@
1
+ <<END0+<<END1+<<END2+"kdsjfasdjf"
2
+ #{m0}
3
+ END0
4
+ #{m1}
5
+ END1
6
+ #{m2}
7
+ END2
8
+
9
+ foo a=<<a,b=<<b,c=<<c
10
+ jfksdkjf
11
+ dkljjkf
12
+ a
13
+ kdljfjkdg
14
+ dfglkdfkgjdf
15
+ dkf
16
+ b
17
+ lkdffdjksadhf
18
+ sdflkdjgsfdkjgsdg
19
+ dsfg;lkdflisgffd
20
+ g
21
+ c
22
+
23
+ foo(a=<<a,b=<<b,c=<<c)
24
+ jfksdkjf
25
+ dkljjkf
26
+ a
27
+ kdljfjkdg
28
+ dfglkdfkgjdf
29
+ dkf
30
+ b
31
+ lkdffdjksadhf
32
+ sdflkdjgsfdkjgsdg
33
+ dsfg;lkdflisgffd
34
+ g
35
+ c
36
+
37
+ def foo(a=<<a,b=<<b,c=<<c)
38
+ jfksdkjf
39
+ dkljjkf
40
+ a
41
+ kdljfjkdg
42
+ dfglkdfkgjdf
43
+ dkf
44
+ b
45
+ lkdffdjksadhf
46
+ sdflkdjgsfdkjgsdg
47
+ dsfg;lkdflisgffd
48
+ g
49
+ c
50
+
51
+ a+b+c
52
+
53
+ end
54
+
55
+ p <<stuff+'foobar'.tr('j-l','d-f')\
56
+ +"more stuff"
57
+ 12345678
58
+ the quick brown fox jumped over the lazy dog
59
+ stuff
60
+
61
+ p <<simple1
62
+ foo
63
+ bar
64
+ baz
65
+ simple1
66
+ p(<<simple2)
67
+ foo2
68
+ bar
69
+ baz
70
+ simple2
71
+
72
+ p <<oof+"gfert"
73
+ #{gleeble #breakpoint
74
+ }
75
+ oof
76
+
77
+ funcnames.collect{|fn| <<-endeval; hn }.to_s(fn,hn,gn=1)
78
+ #{fn} #{gn} #{hn=2}
79
+ #$fn #@gn #@@hn=2
80
+ endeval
81
+ p fn,gn,hn
82
+
83
+ p <<foo+'crazy'+<<bar+'nutty'
84
+ sdsdf
85
+ foo
86
+ sdfsdfgsdfsdf
87
+ bar
88
+
89
+ p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'
90
+ sdsdf
91
+ foo
92
+ sdfsdfgsdfsdf
93
+ bar
94
+ uityeiorwpieo
95
+ baz
96
+
97
+ p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'
98
+ sdsdf
99
+ foo
100
+ sdfsdfgsdfsdf
101
+ bar
102
+ uityeiorwpieo
103
+ baz
104
+ CVBNXVBdfg
105
+ quux
106
+
107
+ p <<foo+'crazy'+<<bar+'nutty'+<<baz+'insane!'+<<quux+'super-wacky'+
108
+ sdsdf
109
+ foo
110
+ sdfsdfgsdfsdf
111
+ bar
112
+ uityeiorwpieo
113
+ baz
114
+ CVBNXVBdfg
115
+ quux
116
+ "5645647585735"
117
+
118
+ p <<foo+
119
+ bar
120
+ baz
121
+ foo
122
+ "quux"
123
+
124
+
125
+ p <<f+<<f
126
+ s435fg
127
+ f
128
+ 2345gdf
129
+ f
130
+
131
+ p <<p
132
+ q\
133
+ p
134
+ p
135
+
136
+ p <<''
137
+ q\
138
+
139
+ p
140
+