rubylexer 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/History.txt +90 -0
  2. data/Manifest.txt +54 -3
  3. data/README.txt +4 -7
  4. data/Rakefile +3 -2
  5. data/lib/rubylexer.rb +856 -323
  6. data/lib/rubylexer/0.7.0.rb +11 -2
  7. data/lib/rubylexer/0.7.1.rb +2 -0
  8. data/lib/rubylexer/charhandler.rb +4 -4
  9. data/lib/rubylexer/context.rb +86 -9
  10. data/lib/rubylexer/rulexer.rb +455 -101
  11. data/lib/rubylexer/token.rb +166 -43
  12. data/lib/rubylexer/tokenprinter.rb +16 -8
  13. data/lib/rubylexer/version.rb +1 -1
  14. data/rubylexer.vpj +98 -0
  15. data/test/code/all_the_gems.rb +33 -0
  16. data/test/code/all_the_raas.rb +226 -0
  17. data/test/code/all_the_rubies.rb +2 -0
  18. data/test/code/deletewarns.rb +19 -1
  19. data/test/code/dumptokens.rb +39 -8
  20. data/test/code/errscan +2 -0
  21. data/test/code/isolate_error.rb +72 -0
  22. data/test/code/lexloop +14 -0
  23. data/test/code/locatetest.rb +150 -8
  24. data/test/code/regression.rb +109 -0
  25. data/test/code/rubylexervsruby.rb +53 -15
  26. data/test/code/strgen.rb +138 -0
  27. data/test/code/tarball.rb +144 -0
  28. data/test/code/testcases.rb +11 -0
  29. data/test/code/tokentest.rb +115 -24
  30. data/test/data/__eof2.rb +1 -0
  31. data/test/data/__eof5.rb +2 -0
  32. data/test/data/__eof6.rb +2 -0
  33. data/test/data/cvtesc.rb +17 -0
  34. data/test/data/g.rb +6 -0
  35. data/test/data/hd0.rb +3 -0
  36. data/test/data/hdateof.rb +2 -0
  37. data/test/data/hdempty.rb +3 -0
  38. data/test/data/hdr.rb +9 -0
  39. data/test/data/hdr_dos.rb +13 -0
  40. data/test/data/hdr_dos2.rb +18 -0
  41. data/test/data/heart.rb +2 -0
  42. data/test/data/here_escnl.rb +25 -0
  43. data/test/data/here_escnl_dos.rb +20 -0
  44. data/test/data/here_squote.rb +3 -0
  45. data/test/data/heremonsters.rb +140 -0
  46. data/test/data/heremonsters.rb.broken +68 -0
  47. data/test/data/heremonsters.rb.broken.save +68 -0
  48. data/test/data/heremonsters_dos.rb +140 -0
  49. data/test/data/heremonsters_dos.rb.broken +68 -0
  50. data/test/data/illegal_oneliners.rb +1 -0
  51. data/test/data/illegal_stanzas.rb +0 -0
  52. data/test/data/make_ws_strdelim.rb +22 -0
  53. data/test/data/maven2_builer_test.rb +82 -0
  54. data/test/data/migration.rb +8944 -0
  55. data/test/data/modl.rb +6 -0
  56. data/test/data/modl_dos.rb +7 -0
  57. data/test/data/modl_fails.rb +10 -0
  58. data/test/data/multilinestring.rb +6 -0
  59. data/test/data/oneliners.rb +555 -0
  60. data/test/data/p-op.rb +2 -0
  61. data/test/data/p.rb +3 -1710
  62. data/test/data/s.rb +90 -21
  63. data/test/data/simple.rb +1 -0
  64. data/test/data/simple_dos.rb +1 -0
  65. data/test/data/stanzas.rb +1194 -0
  66. data/test/data/strdelim_crlf.rb +6 -0
  67. data/test/data/stuff.rb +6 -0
  68. data/test/data/stuff2.rb +5 -0
  69. data/test/data/stuff3.rb +6 -0
  70. data/test/data/stuff4.rb +6 -0
  71. data/test/data/tkweird.rb +20 -0
  72. data/test/data/unending_stuff.rb +5 -0
  73. data/test/data/whatnot.rb +8 -0
  74. data/test/data/ws_strdelim.rb +0 -0
  75. data/test/test.sh +239 -0
  76. data/testing.txt +39 -50
  77. metadata +110 -12
  78. data/test/code/dl_all_gems.rb +0 -43
  79. data/test/code/unpack_all_gems.rb +0 -15
  80. data/test/data/gemlist.txt +0 -280
data/test/code/errscan ADDED
@@ -0,0 +1,2 @@
1
+ #!/bin/sh
2
+ egrep -B1 -v '^(\(expected\) |executing)' locatetest.out
@@ -0,0 +1,72 @@
1
+ require 'test/code/rubylexervsruby'
2
+ opts = GetoptLong.new( [ '--ends', '-e', GetoptLong::OPTIONAL_ARGUMENT] )
3
+ endcount=0
4
+ opts.each {|opt,arg|
5
+ if opt=='--ends'
6
+ if arg==''
7
+ endcount=1
8
+ else
9
+ endcount=arg.to_i
10
+ end
11
+ end
12
+ }
13
+
14
+ ends="end\n"*endcount
15
+
16
+ endcode=ends#+"__END__\n"
17
+
18
+ lines=IO.readlines(ARGV.first)
19
+
20
+ low=1
21
+ high=lines.size-1
22
+
23
+ class DontEvalThis<Exception; end
24
+
25
+ def syntax_ok?(code)
26
+ caught=
27
+ begin
28
+ catch :dontevalthis do
29
+ eval "
30
+ BEGIN{throw :dontevalthis, :notevaled}
31
+ BEGIN{raise DontEvalThis}
32
+ "+code
33
+ end
34
+ rescue DontEvalThis:
35
+ puts "first level eval barrier failed!!!"
36
+ caught=:notevaled
37
+ rescue Exception:
38
+ caught=:unexpected
39
+ end
40
+ case caught
41
+ when :unexpected
42
+ return false
43
+ when :notevaled
44
+ return true
45
+ else #wtf?
46
+ puts "eval barriers totally bypassed?!?!?"
47
+ end
48
+ end
49
+
50
+ span=9999999999999999999999999999
51
+ loop do
52
+ break if high-low>=span
53
+ span=high-low
54
+ adjust=1
55
+ mid=(low+high)/2
56
+ begin
57
+ text=lines.dup
58
+ realmid=mid+adjust
59
+ realmid=[[realmid,low].max,high].min
60
+ text=text[0...realmid] << endcode
61
+ text=text.to_s
62
+ adjust*=-2
63
+ end until syntax_ok?text
64
+
65
+ if RubyLexerVsRuby.rubylexervsruby(ARGV.first+".chunk#{low..high}", text)
66
+ low=realmid
67
+ else
68
+ high=realmid
69
+ end
70
+ end
71
+
72
+ p low..high
data/test/code/lexloop ADDED
@@ -0,0 +1,14 @@
1
+ ruby -Ilib test/code/tokentest.rb --loop --maxws \
2
+ /home/caleb/sandbox/rubyparser/babyparser.rb \
3
+ /home/caleb/sandbox/rubyparser/hd2.rb \
4
+ /home/caleb/sandbox/rubyparser/test_rubyparser.rb \
5
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/rulexer.rb \
6
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/tokenprinter.rb \
7
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/0.6.2.rb \
8
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/0.7.1.rb \
9
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/charset.rb \
10
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/symboltable.rb \
11
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/charhandler.rb \
12
+ /home/caleb/sandbox/rubylexer/lib/rubylexer/0.7.0.rb \
13
+ /home/caleb/sandbox/rubylexer/lib/assert.rb \
14
+ >/dev/null 2>&1
@@ -1,6 +1,30 @@
1
+ =begin legal crap
2
+ rubylexer - a ruby lexer written in ruby
3
+ Copyright (C) 2004,2005,2008 Caleb Clausen
4
+
5
+ This library is free software; you can redistribute it and/or
6
+ modify it under the terms of the GNU Lesser General Public
7
+ License as published by the Free Software Foundation; either
8
+ version 2.1 of the License, or (at your option) any later version.
9
+
10
+ This library is distributed in the hope that it will be useful,
11
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
+ Lesser General Public License for more details.
14
+
15
+ You should have received a copy of the GNU Lesser General Public
16
+ License along with this library; if not, write to the Free Software
17
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
+ =end
19
+
1
20
  require 'test/code/rubylexervsruby'
21
+ require 'test/code/strgen'
22
+
23
+ class RubyLexer
24
+ module LocateTest
2
25
  #ENV['RUBY']||='ruby'
3
26
  $RUBY=ENV['RUBY']||'ruby'
27
+ skip_til=ENV['SKIP_TIL']
4
28
  #test $RUBY || export RUBY=ruby
5
29
 
6
30
  #$RUBYLEXERVSRUBY="#$RUBY test/code/rubylexervsruby.rb"
@@ -31,19 +55,137 @@ ruby -e 'print ($:.sort.uniq+[""]).join"\n"'|xargs -i ls "{}/*.rb" >> test/resul
31
55
 
32
56
  =end
33
57
 
34
- for i in [
35
- RLROOT+"/test/data/p.rb", *Dir["test/data/*.rb"]+`(locate /tk.rb;
36
- locate examples/examples_test.rb;locate .rb; locate rakefile;
37
- locate Rakefile; locate RAKEFILE)|egrep -v '/test/(results|data)/'`.
38
- split("\n")
39
- ] do
58
+ bindirs=ENV['PATH'].split(':')+['/sbin','/usr/sbin']
59
+
60
+
61
+ puts "hack hack hack: absolute path in ruby test file list"
62
+ RUBYBINS=
63
+ bindirs.map{|dir| Dir[dir+"/*"].select{|prog|
64
+ line1=File.open(prog){|f| f.readline} rescue next
65
+ %r{\A#!([^\s]*/env )?[^\s]*/j?ruby(?:1\.[6789])(?:\s|\n)}===line1
66
+ }}.flatten
67
+ TEST_THESE_FIRST=
68
+ %w[
69
+
70
+
71
+
72
+
73
+
74
+
75
+
76
+
77
+
78
+
79
+
80
+
81
+ /home/caleb/sandbox/rubylexer/jewels/xrefresh-server-0.1.0/lib/xrefresh-server.rb
82
+ /home/caleb/sandbox/rubylexer/jewels/shattered-0.7.0/test/acceptance/lib/mesh_test.rb
83
+ /home/caleb/sandbox/rubylexer/jewels/rutils-0.2.3/lib/countries/countries.rb
84
+ /home/caleb/sandbox/rubylexer/jewels/ruport-1.6.1/examples/trac_ticket_status.rb
85
+ /home/caleb/sandbox/rubylexer/jewels/rubysdl-2.0.1/extconf.rb
86
+ /home/caleb/sandbox/rubylexer/jewels/rubysdl-1.1.0/extconf.rb
87
+ /home/caleb/sandbox/rubylexer/jewels/ruby-msg-1.3.1/lib/msg/rtf.rb
88
+ /home/caleb/sandbox/rubylexer/jewels/ruby-mediawiki-0.1/apps/iso_639_leecher.rb
89
+ /home/caleb/sandbox/rubylexer/jewels/ruby-finance-0.2.2/lib/finance/quote/yahoo/australia.rb
90
+ /home/caleb/sandbox/rubylexer/jewels/roby-0.7.2/test/test_task.rb
91
+ /home/caleb/sandbox/rubylexer/jewels/reve-0.0.94/test/test_reve.rb
92
+ /home/caleb/sandbox/rubylexer/jewels/remote_api-0.2.1/test/spec_test.rb
93
+ /home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/getartist.rb
94
+ /home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/getlabel.rb
95
+ /home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/gettrack.rb
96
+ /home/caleb/sandbox/rubylexer/jewels/railscart-0.0.4/starter_app/vendor/plugins/engines/init.rb
97
+ /home/caleb/sandbox/rubylexer/jewels/ok-extensions-1.0.15/test/extensions/test_object.rb
98
+ /home/caleb/sandbox/rubylexer/jewels/oai-0.0.8/lib/oai/harvester/logging.rb
99
+ /home/caleb/sandbox/rubylexer/jewels/oai-0.0.8/examples/providers/dublin_core.rb
100
+ /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/assert-text-equal.rbc
101
+ /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-engines.rbc
102
+ /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-erubis.rbc
103
+ /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-users-guide.rbc
104
+ /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test.rbc
105
+ /home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/testutil.rbc
106
+ /home/caleb/sandbox/rubylexer/jewels/fb-0.5.5/test/CursorTestCases.rb
107
+ /home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/examples/auth.rb
108
+ /home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/examples/upload.rb
109
+ /home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/test/test.rb
110
+ /home/caleb/sandbox/rubylexer/jewels/fox-tool-0.10.0-preview/fox-tool/examples/input.rbin
111
+ /home/caleb/sandbox/rubylexer/jewels/fox-tool-0.10.0-preview/fox-tool/examples/cvs/Base/print.rbin
112
+ /home/caleb/sandbox/rubylexer/jewels/foxGUIb_1.0.0/foxguib_1.0.0/foxguib/src/gui/_guib_genruby.rbin
113
+ /home/caleb/sandbox/rubylexer/jewels/hpricot_scrub-0.3.2/test/scrubber_data.rb
114
+ /home/caleb/sandbox/rubylexer/jewels/htmltools-1.10/test/tc_stacking-parser.rb
115
+ /home/caleb/sandbox/rubylexer/jewels/ludy-0.1.13/test/deprecated/ts_ludy.rb
116
+ /home/caleb/sandbox/rubylexer/jewels/menu_helper-0.0.5/test/unit/menu_test.rb
117
+ /home/caleb/sandbox/rubylexer/jewels/mod_spox-0.0.5/data/mod_spox/extras/PhpCli.rb
118
+ /home/caleb/sandbox/rubylexer/jewels/motiro-0.6.11/app/core/wiki_page_not_found.rb
119
+ /home/caleb/sandbox/rubylexer/jewels/motiro-0.6.11/vendor/plugins/globalize/generators/globalize/templates/migration.rb.gz
120
+
121
+ ]+[
122
+ "/home/caleb/sandbox/rubylexer/jewels/core_ex-0.6.6.3/lib/core_ex/numeric.rb",
123
+ "/home/caleb/sandbox/rubylexer/jewels/cerberus-0.3.6/test/bjam_builder_test.rb",
124
+ "/home/caleb/sandbox/rubylexer/jewels/cerberus-0.3.6/test/maven2_builer_test.rb",
125
+ "/home/caleb/sandbox/rubylexer/jewels/buildr-1.3.1.1/lib/buildr/java/groovyc.rb",
126
+ "/home/caleb/sandbox/rubylexer/jewels/adhearsion-0.7.7/apps/default/helpers/micromenus.rb",
127
+ "/home/caleb/rubies/ruby-1.8.7/instruby.rb",
128
+ "/home/caleb/sandbox/rubylexer/jewels/RuCodeGen-0.3.1/lib/rucodegen/value_incapsulator.rb",
129
+
130
+ "/home/caleb/sandbox/rubylexer/jewels/Wiki2Go-1.17.3/test/TestWiki2GoServlet.rb",
131
+ "/home/caleb/rubies/ruby-1.8.7/test/rss/test_parser_atom_entry.rb",
132
+ "/home/caleb/sandbox/rubylexer/jewels/active_form-0.0.8/test/elements/test_base_element.rb",
133
+ "/home/caleb/sandbox/rubylexer/jewels/dohruby-0.2.1/bin/create_database.rb",
134
+ "/home/caleb/sandbox/rubylexer/jewels/depager-0.2.2/examples/c89/c89.tab.rb",
135
+
136
+ "/home/caleb/sandbox/rubylexer/jewels/samizdat-0.6.1/samizdat/lib/samizdat/storage.rb",
137
+ "/home/caleb/sandbox/rubylexer/jewels/math3d-0.04/tests/make_tests.rb",
138
+ "/home/caleb/sandbox/rubylexer/jewels/QuickBaseClient.rb/quickbasecontactsappbuilder.rb",
139
+ "/home/caleb/sandbox/rubylexer/jewels/QuickBaseClient.rb/quickbaseclient.rb",
140
+ "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._000-IVYTranslater.rb",
141
+ "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._002-ApplicationList.rb",
142
+ "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._001-UnBind.rb",
143
+ "/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/._extconf.rb",
144
+ "/home/caleb/sandbox/rubylexer/jewels/smf-0.15.10/sample/virtual-samp.rb",
145
+ "/home/caleb/sandbox/rubylexer/jewels/syntax/syntax.rb",
146
+ "/home/caleb/sandbox/rubylexer/jewels/rex-1.0rc1/rex/packages/rex/test/rex-20060511.rb",
147
+ "/home/caleb/sandbox/rubylexer/jewels/rex-1.0rc1/rex/packages/rex/test/rex-20060125.rb",
148
+ # "/home/caleb/sandbox/rubylexer/jewels/japanese-zipcodes-0.0.20080227/lib/japanese/zipcodes.rb", #huge!!!!!
149
+ ]
150
+ RUBYLIST=TEST_THESE_FIRST+RUBYBINS+
151
+ [RLROOT+"/test/data/p.rb",
152
+ *Dir["test/data/*.rb"]+
153
+ `(
154
+ locate /inline.rb /tk.rb;
155
+ locate examples/examples_test.rb;
156
+ locate .rb;
157
+ locate rakefile; locate Rakefile; locate RAKEFILE;
158
+ )|egrep -v '(/test/(results|data)/|.broken$)'`.
159
+ split("\n").reject{|i| %r(japanese/zipcodes\.rb\Z)===i }
160
+ ]
161
+ RUBYLIST.uniq!
162
+ if skip_til
163
+ skip_til=RUBYLIST.index(skip_til)
164
+ skip_til or fail "SKIP_TIL not found in list of rubies"
165
+ RUBYLIST.slice! 0...skip_til
166
+ end
167
+ def self.main
168
+ for i in RUBYLIST do
40
169
  # system $RUBYLEXERVSRUBY, i
41
170
  #hmm, rubylexervsruby needs to be upgraded to not regard an output
42
171
  #consisting entirely of warnings as a failure.
43
172
  #if no 'warning' (in any capitalization) for 4 or more lines
44
- RubyLexerVsRuby.rubylexervsruby i #or fail "failed in #{i}"
173
+ begin
174
+ # puts File.readlines("/proc/#{$$}/status").grep(/^VmSize:\s/)
175
+ File.exist? i or next
176
+ RubyLexerVsRuby.rubylexervsruby i,nil,nil,true #or puts "syntax error in #{i}"
177
+ rescue Interrupt; raise
178
+ rescue Exception=>e
179
+ puts "error in: "+i
180
+ puts e
181
+ puts e.backtrace.map{|s| " from: "+s}.join("\n")
182
+ end
183
+ end
45
184
  end
46
-
47
185
  #for i in test/data/p.rb `(locate /tk.rb;locate examples/examples_test.rb;#locate .rb; locate rakefile; locate Rakefile; locate RAKEFILE)|egrep -v '/#test/results/'`; do
48
186
  # $RUBYLEXERVSRUBY $i
49
187
  #done
188
+ end
189
+ end
190
+
191
+ RubyLexer::LocateTest.main if $0==__FILE__
@@ -0,0 +1,109 @@
1
+ require 'open3'
2
+
3
+ require 'test/code/testcases'
4
+ require 'test/code/rubylexervsruby'
5
+ SEP='';
6
+ 'caleb clausen'.each_byte{|ch| SEP<<ch.to_s(2).gsub('0','-').gsub('1','+')}
7
+ SEP<<'(0)'
8
+
9
+ #require 'test/code/rubyoracle'
10
+ class<<RubyLexerVsRuby
11
+
12
+ def progress ruby,input; end
13
+ =begin oracular version was just a bad idea....
14
+ def ruby_parsedump(input,output,ruby)
15
+ #todo: use ruby's md5 lib
16
+ #recursive ruby call here is unavoidable because -y flag has to be set
17
+
18
+ @oracle||= Open3.popen3("#{ruby} -w -y -c")
19
+ @oracle[0].write IO.read(input)
20
+ @oracle[0].flush
21
+ #timeout=Time.now+0.1
22
+ data=''
23
+ while data.empty? # and Time.now<timeout
24
+ begin
25
+ data<< chunk=@oracle[2].read_nonblock(1024)
26
+ timeout=Time.now+0.02
27
+ rescue EOFError
28
+ # data<<"\nError: premature eof...\n"
29
+ break
30
+ rescue Errno::EAGAIN
31
+ break if data[/^Reading a token: \Z/] and Time.now>=timeout
32
+ end while chunk
33
+ end
34
+
35
+ status=0
36
+ lines=data.split("\n")
37
+ File.open(output,"w") { |outfd|
38
+ lines.each{|line|
39
+ outfd.puts(line) if /^Shifting/===line
40
+ if /^#{DeleteWarns::WARNERRREX}|^Error|^(Now at end of input\.)/o===line
41
+ outfd.puts(line)
42
+ if status.zero? and $2!="warning"
43
+ status=1 unless $4 #unless end of input seen
44
+ @oracle.each{|fd| fd.close} if @oracle
45
+ @oracle=nil
46
+ end
47
+ end
48
+ }
49
+ }
50
+ return status
51
+ end
52
+ =end
53
+ end
54
+
55
+ if __FILE__==$0
56
+ require 'test/unit'
57
+
58
+ class LexerTests<Test::Unit::TestCase
59
+ class LexerTestFailure<RuntimeError; end
60
+ class DifferencesFromMRILex<LexerTestFailure; end
61
+
62
+ i=-1
63
+ test_code= TestCases::TESTCASES.map{|tc|
64
+ i+=1
65
+ name="testcase_#{i}__"
66
+ esctc=tc.gsub(/['\\]/){"\\"+$&}
67
+ %[
68
+ define_method '#{name}' do
69
+ difflines=[]
70
+ begin
71
+ res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines)
72
+ difflines.empty? or raise DifferencesFromMRILex, difflines
73
+ res or raise LexerTestFailure, ''
74
+ rescue Interrupt: exit
75
+ rescue Exception=>e:
76
+ e.message<<"\n"+'while lexing: #{esctc}'
77
+ raise e
78
+ end
79
+ end
80
+ ]
81
+ }.to_s
82
+
83
+ illegal_test_code= TestCases::ILLEGAL_TESTCASES.map{|tc|
84
+ i+=1
85
+ name="testcase_#{i}__"
86
+ esctc=tc.gsub(/['\\]/){"\\"+$&}
87
+ %[
88
+ define_method '#{name}' do
89
+ difflines=[]
90
+ begin
91
+ res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines)
92
+ difflines.empty? or raise DifferencesFromMRILex, difflines
93
+ res or raise LexerTestFailure, ''
94
+ rescue LexerTestFailure:
95
+ puts 'warning: test failure lexing "#{esctc}"'
96
+ rescue Interrupt: exit
97
+ rescue Exception=>e:
98
+ e.message<<"\n"+'while lexing: #{esctc}'
99
+ raise e
100
+ end
101
+ end
102
+ ]
103
+ }.to_s
104
+
105
+
106
+ # puts test_code+illegal_test_code
107
+ eval test_code+illegal_test_code
108
+ end
109
+ end
@@ -1,4 +1,22 @@
1
1
  #!/usr/bin/ruby
2
+ =begin legal crap
3
+ rubylexer - a ruby lexer written in ruby
4
+ Copyright (C) 2004,2005,2008 Caleb Clausen
5
+
6
+ This library is free software; you can redistribute it and/or
7
+ modify it under the terms of the GNU Lesser General Public
8
+ License as published by the Free Software Foundation; either
9
+ version 2.1 of the License, or (at your option) any later version.
10
+
11
+ This library is distributed in the hope that it will be useful,
12
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14
+ Lesser General Public License for more details.
15
+
16
+ You should have received a copy of the GNU Lesser General Public
17
+ License along with this library; if not, write to the Free Software
18
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
+ =end
2
20
  #$DEBUG=$VERBOSE=true
3
21
  $Debug=true
4
22
  require "getoptlong"
@@ -39,15 +57,20 @@ def ruby_parsedump(input,output,ruby)
39
57
  }
40
58
  }
41
59
  ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
60
+ return status>>8
42
61
  end
43
62
 
44
63
  def head(fname)
45
- print "foobaaaaaaaaar\n\n\n\n\n\n"
46
64
  File.open(fname){|fd| print(fd.read(512)+"\n") }
47
65
  end
48
66
 
49
- def rubylexervsruby(input,stringdata=nil,&ignore_it)
50
67
 
68
+ def progress ruby,input
69
+ print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
70
+ $stdout.flush
71
+ end
72
+
73
+ def rubylexervsruby(input,stringdata=nil,difflines=nil,bulk=nil,&ignore_it)
51
74
  #cmdpath= `which #$0`
52
75
  cmddir=Dir.getwd+"/test/code/"
53
76
  base='test/results/'+File.basename(input)
@@ -67,12 +90,14 @@ expected_failures=Dir.getwd+"/test/code/"+File.basename(input)+".expected_failur
67
90
 
68
91
  nop_ruby "#{input[/\.gz$/]&&'z'}cat", input, nopfile, stringdata
69
92
 
70
- print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
71
-
72
- ruby_parsedump nopfile, origfile, ruby
73
-
74
93
 
94
+ legal=ruby_parsedump nopfile, origfile, ruby
95
+ if false&&legal.nonzero?
96
+ puts "skipping #{input}; not legal"
97
+ return
98
+ end
75
99
 
100
+ progress ruby, input
76
101
  tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new, nil, _ttfile
77
102
  tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new(' '), nil, mttfile
78
103
 
@@ -81,7 +106,7 @@ ruby_parsedump _ttfile, p_ttfile, ruby
81
106
  ruby_parsedump mttfile, pmttfile, ruby
82
107
 
83
108
  if File.exists?(p_ttfile)
84
- IO.popen("diff -u1 -b #{origfile} #{p_ttfile}"){ |pipe|
109
+ IO.popen("diff --unified=1 -b #{origfile} #{p_ttfile}"){ |pipe|
85
110
  File.open(p_ttdiff,"w") { |diff|
86
111
  DeleteWarns.deletewarns(pipe){|s| diff.print s}
87
112
  }
@@ -90,7 +115,7 @@ if File.exists?(p_ttfile)
90
115
  end
91
116
 
92
117
  if File.exists?(pmttfile)
93
- IO.popen("diff -u1 -b #{origfile} #{pmttfile}"){ |pipe|
118
+ IO.popen("diff --unified=1 -b #{origfile} #{pmttfile}"){ |pipe|
94
119
  File.open(pmttdiff,"w") { |diff|
95
120
  DeleteWarns.deletewarns(pipe){|s| diff.print s}
96
121
  }
@@ -101,10 +126,11 @@ end
101
126
  list=[]
102
127
  #nonwarn4=/(^(?![^\n]*warning[^\n]*)[^\n]*\n){4,}/im
103
128
  #4 or more non-warning lines:
104
- nonwarn4=/^(?:(?![^\r\n]*warning)[^\r\n]+(?:\r\n?|\n\r?)){4,}/mi
129
+ nonwarn4=/^(?:(?![^\r\n]*warning)[^\n]+\n){4,}/mi
105
130
  result=true
131
+ (system "ruby -c #{input} >/dev/null 2>&1" or expected="(expected) ") unless stringdata
106
132
  for name in [p_ttdiff,pmttdiff] do
107
- i=File.read(name)
133
+ i=File.read(name) rescue next
108
134
  # i.tr("\r","\n")
109
135
  # i.gsub!(/^\n/m, '')
110
136
  i.sub!(/\A([^\r\n]+(\r\n?|\n\r?)){2}/, '') #remove 1st 2 lines
@@ -116,9 +142,10 @@ for name in [p_ttdiff,pmttdiff] do
116
142
 
117
143
  unless list.empty?
118
144
  list=list.join("\n") +"\n"
119
- unless (File.exists?(expected_failures) and File.read(expected_failures))==list
120
- print list
121
- result=false
145
+ unless (File.exists?(expected_failures) and File.read(expected_failures)==list)
146
+ result=!!expected
147
+ print "#{expected}error in: #{input}\n"
148
+ (difflines ? difflines.push(*list) : print(list)) unless expected
122
149
  end
123
150
  list=[]
124
151
  end
@@ -126,6 +153,8 @@ end
126
153
 
127
154
  #print( list.join("\n") +"\n")
128
155
  #Dir.chdir olddir
156
+ File.unlink(*Dir[base+".{tt,mtt,nop}{,.prs}{,.diff}"]) if result and bulk
157
+
129
158
  return result
130
159
 
131
160
  =begin
@@ -147,10 +176,14 @@ end
147
176
  return false
148
177
  =end
149
178
 
179
+ rescue Interrupt: raise
150
180
  rescue Exception
151
- system "ruby -c #{input} >/dev/null 2>&1" or expected="(expected) "
181
+ File.exist? nopfile and
182
+ system "ruby -c #{nopfile} >/dev/null 2>&1" or expected="(expected) " unless stringdata
152
183
  print "#{expected}error in: #{input}\n"
153
- raise unless expected
184
+ File.unlink(*Dir[base+".{tt,mtt,nop}{,.prs}{,.diff}"]) if expected and bulk
185
+ return true if expected
186
+ raise
154
187
  end
155
188
  end
156
189
 
@@ -165,6 +198,11 @@ opts.each{|opt,arg|
165
198
  }
166
199
 
167
200
  input||=ARGV[0]
201
+
202
+ unless input
203
+ input="-"
204
+ stringdata=$stdin.read
205
+ end
168
206
  RubyLexerVsRuby.rubylexervsruby(input,stringdata) and exit 0
169
207
 
170
208
  exit 1