rubylexer 0.6.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. data/History.txt +55 -0
  2. data/Manifest.txt +67 -0
  3. data/README.txt +103 -0
  4. data/Rakefile +24 -0
  5. data/howtouse.txt +9 -6
  6. data/{assert.rb → lib/assert.rb} +11 -11
  7. data/{rubylexer.rb → lib/rubylexer.rb} +645 -342
  8. data/lib/rubylexer/0.6.2.rb +39 -0
  9. data/lib/rubylexer/0.6.rb +5 -0
  10. data/lib/rubylexer/0.7.0.rb +2 -0
  11. data/{charhandler.rb → lib/rubylexer/charhandler.rb} +4 -2
  12. data/{charset.rb → lib/rubylexer/charset.rb} +4 -3
  13. data/{context.rb → lib/rubylexer/context.rb} +48 -18
  14. data/{rubycode.rb → lib/rubylexer/rubycode.rb} +5 -3
  15. data/{rulexer.rb → lib/rubylexer/rulexer.rb} +180 -102
  16. data/{symboltable.rb → lib/rubylexer/symboltable.rb} +10 -1
  17. data/{token.rb → lib/rubylexer/token.rb} +72 -20
  18. data/{tokenprinter.rb → lib/rubylexer/tokenprinter.rb} +39 -16
  19. data/lib/rubylexer/version.rb +3 -0
  20. data/{testcode → test/code}/deletewarns.rb +0 -0
  21. data/test/code/dl_all_gems.rb +43 -0
  22. data/{testcode → test/code}/dumptokens.rb +12 -9
  23. data/test/code/locatetest +30 -0
  24. data/test/code/locatetest.rb +49 -0
  25. data/test/code/rubylexervsruby.rb +173 -0
  26. data/{testcode → test/code}/tokentest.rb +62 -51
  27. data/{testcode → test/code}/torment +8 -8
  28. data/test/code/unpack_all_gems.rb +15 -0
  29. data/{testdata → test/data}/1.rb.broken +0 -0
  30. data/{testdata → test/data}/23.rb +0 -0
  31. data/test/data/__end__.rb +2 -0
  32. data/test/data/__end__2.rb +3 -0
  33. data/test/data/and.rb +5 -0
  34. data/test/data/blockassigntest.rb +23 -0
  35. data/test/data/chunky.plain.rb +75 -0
  36. data/test/data/chunky_bacon.rb +112 -0
  37. data/test/data/chunky_bacon2.rb +112 -0
  38. data/test/data/chunky_bacon3.rb +112 -0
  39. data/test/data/chunky_bacon4.rb +112 -0
  40. data/test/data/for.rb +45 -0
  41. data/test/data/format.rb +6 -0
  42. data/{testdata → test/data}/g.rb +0 -0
  43. data/test/data/gemlist.txt +280 -0
  44. data/test/data/heart.rb +7 -0
  45. data/test/data/if.rb +6 -0
  46. data/test/data/jarh.rb +369 -0
  47. data/test/data/lbrace.rb +4 -0
  48. data/test/data/lbrack.rb +4 -0
  49. data/{testdata → test/data}/newsyntax.rb +0 -0
  50. data/{testdata → test/data}/noeolatend.rb +0 -0
  51. data/test/data/p-op.rb +8 -0
  52. data/{testdata → test/data}/p.rb +671 -79
  53. data/{testdata → test/data}/pleac.rb.broken +0 -0
  54. data/{testdata → test/data}/pre.rb +0 -0
  55. data/{testdata → test/data}/pre.unix.rb +0 -0
  56. data/{testdata → test/data}/regtest.rb +0 -0
  57. data/test/data/rescue.rb +35 -0
  58. data/test/data/s.rb +186 -0
  59. data/test/data/strinc.rb +2 -0
  60. data/{testdata → test/data}/tokentest.assert.rb.can +0 -0
  61. data/test/data/untermed_here.rb.broken +2 -0
  62. data/test/data/untermed_string.rb.broken +1 -0
  63. data/{testdata → test/data}/untitled1.rb +0 -0
  64. data/{testdata → test/data}/w.rb +0 -0
  65. data/{testdata → test/data}/wsdlDriver.rb +0 -0
  66. data/testing.txt +6 -4
  67. metadata +163 -59
  68. data/README +0 -134
  69. data/Rantfile +0 -37
  70. data/io.each_til_charset.rb +0 -247
  71. data/require.rb +0 -103
  72. data/rlold.rb +0 -12
  73. data/testcode/locatetest +0 -12
  74. data/testcode/rubylexervsruby.rb +0 -104
  75. data/testcode/rubylexervsruby.sh +0 -51
  76. data/testresults/placeholder +0 -0
data/rlold.rb DELETED
@@ -1,12 +0,0 @@
1
- class RubyLexer06 < RubyLexer
2
- alias lexerror lexerror_exception
3
- end
4
-
5
- class<<RubyLexer
6
- def version(num)
7
- case num
8
- when 0.0..0.6: RubyLexer06
9
- else RubyLexer
10
- end
11
- end
12
- end
data/testcode/locatetest DELETED
@@ -1,12 +0,0 @@
1
- test $RUBY || export RUBY=ruby
2
- export RUBYLEXERVSRUBY="$RUBY testcode/rubylexervsruby.rb"
3
-
4
- if $RUBY --version|grep '^ruby 1\.6'; then
5
- echo 'error: need ruby 1.8'; exit
6
- fi
7
-
8
-
9
-
10
- for i in testdata/p.rb `(locate tk.rb;locate examples/examples_test.rb;locate .rb)|egrep -v '/testresults/|files/patch-'`; do
11
- $RUBYLEXERVSRUBY $i
12
- done
@@ -1,104 +0,0 @@
1
- #!/usr/bin/ruby -dw
2
-
3
- require "testcode/tokentest"
4
- require "testcode/deletewarns"
5
-
6
- ENABLEMD5=false
7
-
8
- #these remain globals only for ease in interpolation
9
- $cmdpath= `which #$0`
10
- $cmddir=File.dirname $cmdpath
11
- $input=ARGV[0]
12
- $base='testresults/'+File.basename($input)
13
- $_ttfile=$base+'.tt'
14
- $mttfile=$base+'.mtt'
15
- $p_ttfile=$_ttfile+'.prs'
16
- $pmttfile=$mttfile+'.prs'
17
- $p_ttdiff=$p_ttfile+'.diff'
18
- $pmttdiff=$pmttfile+'.diff'
19
- $nopfile=$base+'.nop'
20
- $origfile=$nopfile+'.prs'
21
- $ruby=ENV['RUBY'] || 'ruby'
22
-
23
- def nop_ruby(cmd,input,output)
24
- File.open(output,'w'){|f| f.print "BEGIN{exit};\n" }
25
- system [cmd,input,'>>',output].join(' ')
26
- end
27
-
28
- def ruby_parsedump(input,output)
29
- #todo: use ruby's md5 lib
30
- #recursive ruby call here is unavoidable because -y flag has to be set
31
-
32
- #do nothing if input unchanged
33
- ENABLEMD5 and system "md5sum -c #{input}.md5 2>/dev/null" and return
34
-
35
- status=0
36
- IO.popen("#$ruby -w -y < #{input} 2>&1"){ |pipe| File.open(output,"w") { |outfd|
37
- pipe.each{ |line|
38
- outfd.print(line) \
39
- if /^Shifting|^#{DeleteWarns::WARNERRREX}/o===line
40
- #elsif /(warning|error)/i===line
41
- # raise("a warning or error, appearently, not caught by rex above: "+line)
42
- }
43
- pid,status=Process.waitpid2 pipe.pid #get err status of subprocess
44
- } }
45
- ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
46
- end
47
-
48
- Dir.chdir $cmddir + '/..'
49
-
50
- nop_ruby "#{$input[/\.gz$/]&&'z'}cat", $input, $nopfile
51
-
52
- print "executing: #$ruby testcode/tokentest.rb --keepws #$input\n"
53
-
54
- ruby_parsedump $nopfile, $origfile
55
-
56
-
57
-
58
- tokentest $nopfile, RubyLexer, KeepWsTokenPrinter.new, nil, $_ttfile
59
- tokentest $nopfile, RubyLexer, KeepWsTokenPrinter.new(' '), nil, $mttfile
60
-
61
-
62
- ruby_parsedump $_ttfile, $p_ttfile
63
- ruby_parsedump $mttfile, $pmttfile
64
-
65
- if File.exists?($p_ttfile)
66
- IO.popen("diff -u1 -b #$origfile #$p_ttfile"){ |pipe|
67
- File.open($p_ttdiff,"w") { |diff|
68
- DeleteWarns.deletewarns(pipe){|s| diff.print s}
69
- }
70
- }
71
- File.unlink $p_ttfile
72
- end
73
-
74
- if File.exists?($pmttfile)
75
- IO.popen("diff -u1 -b #$origfile #$pmttfile"){ |pipe|
76
- File.open($pmttdiff,"w") { |diff|
77
- DeleteWarns.deletewarns(pipe){|s| diff.print s}
78
- }
79
- }
80
- File.unlink $pmttfile
81
- end
82
-
83
- def head(fname)
84
- File.open(fname){|fd| 10.times{ print((fd.gets or break)) } }
85
- end
86
-
87
- case File.zero?($p_ttdiff).to_s +
88
- File.zero?($pmttdiff).to_s
89
- when 'falsefalse' then
90
- head $p_ttdiff
91
- print "omitting #$pmttdiff\n"
92
- when 'falsetrue'
93
- head $p_ttdiff
94
- when 'truefalse'
95
- head $pmttdiff
96
- when 'truetrue'
97
- #File.unlink $origfile
98
- exit(0)
99
- default
100
- raise "unexpected 2bool val"
101
- end
102
-
103
- exit 1
104
-
@@ -1,51 +0,0 @@
1
- #!/bin/sh
2
- #set -v
3
- function nop_ruby() { (echo "BEGIN{exit};"; exec $1 $2); }
4
- function ruby_parsedump() {
5
- md5sum -c $1.md5 2>/dev/null && return #do nothing if input unchanged
6
- ruby -w -y < $1 2>&1 | grep ^Shift|cut -d" " -f3 >$2
7
- md5sum $1 > $1.md5
8
- }
9
-
10
- cmdpath=`which $0`
11
- cd `dirname $cmdpath`/..
12
-
13
- cmddir=`dirname $cmdpath`
14
- base=testresults/`basename $1`
15
- origfile=$base.nop.prs
16
- _ttfile=$base.tt
17
- mttfile=$base.mtt
18
- p_ttfile=$_ttfile.prs
19
- pmttfile=$mttfile.prs
20
- nopfile=$base.nop
21
-
22
- #if awk "BEGIN{if(\"$1\" ~! /\.gz$/) exit(1)}"; then
23
- if ruby -e "'$1'[/\.gz\$/] or exit 1"; then
24
- nop_ruby zcat $1 > $nopfile ;
25
- else
26
- nop_ruby cat $1 > $nopfile ;
27
- fi
28
-
29
-
30
-
31
- echo executing: testcode/tokentest.rb --ruby --keepws $1 '\|'
32
- testcode/tokentest.rb --ruby --keepws $nopfile > $_ttfile
33
- testcode/tokentest.rb --ruby --maxws $nopfile > $mttfile
34
- ruby_parsedump $nopfile $origfile
35
- ruby_parsedump $_ttfile $p_ttfile
36
- ruby_parsedump $mttfile $pmttfile
37
-
38
- [ -e $p_ttfile ] && (diff -ub $origfile $p_ttfile | \
39
- testcode/deletewarns.rb > $p_ttfile.diff
40
- rm $p_ttfile
41
- )
42
- [ -e $pmttfile ] && (diff -ub $origfile $pmttfile | \
43
- testcode/deletewarns.rb > $pmttfile.diff
44
- rm $pmttfile
45
- )
46
- head $p_ttfile.diff
47
- test -s $p_ttfile.diff || head $pmttfile.diff
48
- test -s $p_ttfile.diff && test -s $pmttfile.diff && echo omitting $pmttfile.diff
49
-
50
- test -z $p_ttfile.diff && test -z $pmttfile.diff && exit 0
51
- exit 1
File without changes