rubylexer 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +55 -0
- data/Manifest.txt +67 -0
- data/README.txt +103 -0
- data/Rakefile +24 -0
- data/howtouse.txt +9 -6
- data/{assert.rb → lib/assert.rb} +11 -11
- data/{rubylexer.rb → lib/rubylexer.rb} +645 -342
- data/lib/rubylexer/0.6.2.rb +39 -0
- data/lib/rubylexer/0.6.rb +5 -0
- data/lib/rubylexer/0.7.0.rb +2 -0
- data/{charhandler.rb → lib/rubylexer/charhandler.rb} +4 -2
- data/{charset.rb → lib/rubylexer/charset.rb} +4 -3
- data/{context.rb → lib/rubylexer/context.rb} +48 -18
- data/{rubycode.rb → lib/rubylexer/rubycode.rb} +5 -3
- data/{rulexer.rb → lib/rubylexer/rulexer.rb} +180 -102
- data/{symboltable.rb → lib/rubylexer/symboltable.rb} +10 -1
- data/{token.rb → lib/rubylexer/token.rb} +72 -20
- data/{tokenprinter.rb → lib/rubylexer/tokenprinter.rb} +39 -16
- data/lib/rubylexer/version.rb +3 -0
- data/{testcode → test/code}/deletewarns.rb +0 -0
- data/test/code/dl_all_gems.rb +43 -0
- data/{testcode → test/code}/dumptokens.rb +12 -9
- data/test/code/locatetest +30 -0
- data/test/code/locatetest.rb +49 -0
- data/test/code/rubylexervsruby.rb +173 -0
- data/{testcode → test/code}/tokentest.rb +62 -51
- data/{testcode → test/code}/torment +8 -8
- data/test/code/unpack_all_gems.rb +15 -0
- data/{testdata → test/data}/1.rb.broken +0 -0
- data/{testdata → test/data}/23.rb +0 -0
- data/test/data/__end__.rb +2 -0
- data/test/data/__end__2.rb +3 -0
- data/test/data/and.rb +5 -0
- data/test/data/blockassigntest.rb +23 -0
- data/test/data/chunky.plain.rb +75 -0
- data/test/data/chunky_bacon.rb +112 -0
- data/test/data/chunky_bacon2.rb +112 -0
- data/test/data/chunky_bacon3.rb +112 -0
- data/test/data/chunky_bacon4.rb +112 -0
- data/test/data/for.rb +45 -0
- data/test/data/format.rb +6 -0
- data/{testdata → test/data}/g.rb +0 -0
- data/test/data/gemlist.txt +280 -0
- data/test/data/heart.rb +7 -0
- data/test/data/if.rb +6 -0
- data/test/data/jarh.rb +369 -0
- data/test/data/lbrace.rb +4 -0
- data/test/data/lbrack.rb +4 -0
- data/{testdata → test/data}/newsyntax.rb +0 -0
- data/{testdata → test/data}/noeolatend.rb +0 -0
- data/test/data/p-op.rb +8 -0
- data/{testdata → test/data}/p.rb +671 -79
- data/{testdata → test/data}/pleac.rb.broken +0 -0
- data/{testdata → test/data}/pre.rb +0 -0
- data/{testdata → test/data}/pre.unix.rb +0 -0
- data/{testdata → test/data}/regtest.rb +0 -0
- data/test/data/rescue.rb +35 -0
- data/test/data/s.rb +186 -0
- data/test/data/strinc.rb +2 -0
- data/{testdata → test/data}/tokentest.assert.rb.can +0 -0
- data/test/data/untermed_here.rb.broken +2 -0
- data/test/data/untermed_string.rb.broken +1 -0
- data/{testdata → test/data}/untitled1.rb +0 -0
- data/{testdata → test/data}/w.rb +0 -0
- data/{testdata → test/data}/wsdlDriver.rb +0 -0
- data/testing.txt +6 -4
- metadata +163 -59
- data/README +0 -134
- data/Rantfile +0 -37
- data/io.each_til_charset.rb +0 -247
- data/require.rb +0 -103
- data/rlold.rb +0 -12
- data/testcode/locatetest +0 -12
- data/testcode/rubylexervsruby.rb +0 -104
- data/testcode/rubylexervsruby.sh +0 -51
- data/testresults/placeholder +0 -0
data/rlold.rb
DELETED
data/testcode/locatetest
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
test $RUBY || export RUBY=ruby
|
2
|
-
export RUBYLEXERVSRUBY="$RUBY testcode/rubylexervsruby.rb"
|
3
|
-
|
4
|
-
if $RUBY --version|grep '^ruby 1\.6'; then
|
5
|
-
echo 'error: need ruby 1.8'; exit
|
6
|
-
fi
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
for i in testdata/p.rb `(locate tk.rb;locate examples/examples_test.rb;locate .rb)|egrep -v '/testresults/|files/patch-'`; do
|
11
|
-
$RUBYLEXERVSRUBY $i
|
12
|
-
done
|
data/testcode/rubylexervsruby.rb
DELETED
@@ -1,104 +0,0 @@
|
|
1
|
-
#!/usr/bin/ruby -dw
|
2
|
-
|
3
|
-
require "testcode/tokentest"
|
4
|
-
require "testcode/deletewarns"
|
5
|
-
|
6
|
-
ENABLEMD5=false
|
7
|
-
|
8
|
-
#these remain globals only for ease in interpolation
|
9
|
-
$cmdpath= `which #$0`
|
10
|
-
$cmddir=File.dirname $cmdpath
|
11
|
-
$input=ARGV[0]
|
12
|
-
$base='testresults/'+File.basename($input)
|
13
|
-
$_ttfile=$base+'.tt'
|
14
|
-
$mttfile=$base+'.mtt'
|
15
|
-
$p_ttfile=$_ttfile+'.prs'
|
16
|
-
$pmttfile=$mttfile+'.prs'
|
17
|
-
$p_ttdiff=$p_ttfile+'.diff'
|
18
|
-
$pmttdiff=$pmttfile+'.diff'
|
19
|
-
$nopfile=$base+'.nop'
|
20
|
-
$origfile=$nopfile+'.prs'
|
21
|
-
$ruby=ENV['RUBY'] || 'ruby'
|
22
|
-
|
23
|
-
def nop_ruby(cmd,input,output)
|
24
|
-
File.open(output,'w'){|f| f.print "BEGIN{exit};\n" }
|
25
|
-
system [cmd,input,'>>',output].join(' ')
|
26
|
-
end
|
27
|
-
|
28
|
-
def ruby_parsedump(input,output)
|
29
|
-
#todo: use ruby's md5 lib
|
30
|
-
#recursive ruby call here is unavoidable because -y flag has to be set
|
31
|
-
|
32
|
-
#do nothing if input unchanged
|
33
|
-
ENABLEMD5 and system "md5sum -c #{input}.md5 2>/dev/null" and return
|
34
|
-
|
35
|
-
status=0
|
36
|
-
IO.popen("#$ruby -w -y < #{input} 2>&1"){ |pipe| File.open(output,"w") { |outfd|
|
37
|
-
pipe.each{ |line|
|
38
|
-
outfd.print(line) \
|
39
|
-
if /^Shifting|^#{DeleteWarns::WARNERRREX}/o===line
|
40
|
-
#elsif /(warning|error)/i===line
|
41
|
-
# raise("a warning or error, appearently, not caught by rex above: "+line)
|
42
|
-
}
|
43
|
-
pid,status=Process.waitpid2 pipe.pid #get err status of subprocess
|
44
|
-
} }
|
45
|
-
ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
|
46
|
-
end
|
47
|
-
|
48
|
-
Dir.chdir $cmddir + '/..'
|
49
|
-
|
50
|
-
nop_ruby "#{$input[/\.gz$/]&&'z'}cat", $input, $nopfile
|
51
|
-
|
52
|
-
print "executing: #$ruby testcode/tokentest.rb --keepws #$input\n"
|
53
|
-
|
54
|
-
ruby_parsedump $nopfile, $origfile
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
tokentest $nopfile, RubyLexer, KeepWsTokenPrinter.new, nil, $_ttfile
|
59
|
-
tokentest $nopfile, RubyLexer, KeepWsTokenPrinter.new(' '), nil, $mttfile
|
60
|
-
|
61
|
-
|
62
|
-
ruby_parsedump $_ttfile, $p_ttfile
|
63
|
-
ruby_parsedump $mttfile, $pmttfile
|
64
|
-
|
65
|
-
if File.exists?($p_ttfile)
|
66
|
-
IO.popen("diff -u1 -b #$origfile #$p_ttfile"){ |pipe|
|
67
|
-
File.open($p_ttdiff,"w") { |diff|
|
68
|
-
DeleteWarns.deletewarns(pipe){|s| diff.print s}
|
69
|
-
}
|
70
|
-
}
|
71
|
-
File.unlink $p_ttfile
|
72
|
-
end
|
73
|
-
|
74
|
-
if File.exists?($pmttfile)
|
75
|
-
IO.popen("diff -u1 -b #$origfile #$pmttfile"){ |pipe|
|
76
|
-
File.open($pmttdiff,"w") { |diff|
|
77
|
-
DeleteWarns.deletewarns(pipe){|s| diff.print s}
|
78
|
-
}
|
79
|
-
}
|
80
|
-
File.unlink $pmttfile
|
81
|
-
end
|
82
|
-
|
83
|
-
def head(fname)
|
84
|
-
File.open(fname){|fd| 10.times{ print((fd.gets or break)) } }
|
85
|
-
end
|
86
|
-
|
87
|
-
case File.zero?($p_ttdiff).to_s +
|
88
|
-
File.zero?($pmttdiff).to_s
|
89
|
-
when 'falsefalse' then
|
90
|
-
head $p_ttdiff
|
91
|
-
print "omitting #$pmttdiff\n"
|
92
|
-
when 'falsetrue'
|
93
|
-
head $p_ttdiff
|
94
|
-
when 'truefalse'
|
95
|
-
head $pmttdiff
|
96
|
-
when 'truetrue'
|
97
|
-
#File.unlink $origfile
|
98
|
-
exit(0)
|
99
|
-
default
|
100
|
-
raise "unexpected 2bool val"
|
101
|
-
end
|
102
|
-
|
103
|
-
exit 1
|
104
|
-
|
data/testcode/rubylexervsruby.sh
DELETED
@@ -1,51 +0,0 @@
|
|
1
|
-
#!/bin/sh
|
2
|
-
#set -v
|
3
|
-
function nop_ruby() { (echo "BEGIN{exit};"; exec $1 $2); }
|
4
|
-
function ruby_parsedump() {
|
5
|
-
md5sum -c $1.md5 2>/dev/null && return #do nothing if input unchanged
|
6
|
-
ruby -w -y < $1 2>&1 | grep ^Shift|cut -d" " -f3 >$2
|
7
|
-
md5sum $1 > $1.md5
|
8
|
-
}
|
9
|
-
|
10
|
-
cmdpath=`which $0`
|
11
|
-
cd `dirname $cmdpath`/..
|
12
|
-
|
13
|
-
cmddir=`dirname $cmdpath`
|
14
|
-
base=testresults/`basename $1`
|
15
|
-
origfile=$base.nop.prs
|
16
|
-
_ttfile=$base.tt
|
17
|
-
mttfile=$base.mtt
|
18
|
-
p_ttfile=$_ttfile.prs
|
19
|
-
pmttfile=$mttfile.prs
|
20
|
-
nopfile=$base.nop
|
21
|
-
|
22
|
-
#if awk "BEGIN{if(\"$1\" ~! /\.gz$/) exit(1)}"; then
|
23
|
-
if ruby -e "'$1'[/\.gz\$/] or exit 1"; then
|
24
|
-
nop_ruby zcat $1 > $nopfile ;
|
25
|
-
else
|
26
|
-
nop_ruby cat $1 > $nopfile ;
|
27
|
-
fi
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
echo executing: testcode/tokentest.rb --ruby --keepws $1 '\|'
|
32
|
-
testcode/tokentest.rb --ruby --keepws $nopfile > $_ttfile
|
33
|
-
testcode/tokentest.rb --ruby --maxws $nopfile > $mttfile
|
34
|
-
ruby_parsedump $nopfile $origfile
|
35
|
-
ruby_parsedump $_ttfile $p_ttfile
|
36
|
-
ruby_parsedump $mttfile $pmttfile
|
37
|
-
|
38
|
-
[ -e $p_ttfile ] && (diff -ub $origfile $p_ttfile | \
|
39
|
-
testcode/deletewarns.rb > $p_ttfile.diff
|
40
|
-
rm $p_ttfile
|
41
|
-
)
|
42
|
-
[ -e $pmttfile ] && (diff -ub $origfile $pmttfile | \
|
43
|
-
testcode/deletewarns.rb > $pmttfile.diff
|
44
|
-
rm $pmttfile
|
45
|
-
)
|
46
|
-
head $p_ttfile.diff
|
47
|
-
test -s $p_ttfile.diff || head $pmttfile.diff
|
48
|
-
test -s $p_ttfile.diff && test -s $pmttfile.diff && echo omitting $pmttfile.diff
|
49
|
-
|
50
|
-
test -z $p_ttfile.diff && test -z $pmttfile.diff && exit 0
|
51
|
-
exit 1
|
data/testresults/placeholder
DELETED
File without changes
|