rubylexer 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +90 -0
- data/Manifest.txt +54 -3
- data/README.txt +4 -7
- data/Rakefile +3 -2
- data/lib/rubylexer.rb +856 -323
- data/lib/rubylexer/0.7.0.rb +11 -2
- data/lib/rubylexer/0.7.1.rb +2 -0
- data/lib/rubylexer/charhandler.rb +4 -4
- data/lib/rubylexer/context.rb +86 -9
- data/lib/rubylexer/rulexer.rb +455 -101
- data/lib/rubylexer/token.rb +166 -43
- data/lib/rubylexer/tokenprinter.rb +16 -8
- data/lib/rubylexer/version.rb +1 -1
- data/rubylexer.vpj +98 -0
- data/test/code/all_the_gems.rb +33 -0
- data/test/code/all_the_raas.rb +226 -0
- data/test/code/all_the_rubies.rb +2 -0
- data/test/code/deletewarns.rb +19 -1
- data/test/code/dumptokens.rb +39 -8
- data/test/code/errscan +2 -0
- data/test/code/isolate_error.rb +72 -0
- data/test/code/lexloop +14 -0
- data/test/code/locatetest.rb +150 -8
- data/test/code/regression.rb +109 -0
- data/test/code/rubylexervsruby.rb +53 -15
- data/test/code/strgen.rb +138 -0
- data/test/code/tarball.rb +144 -0
- data/test/code/testcases.rb +11 -0
- data/test/code/tokentest.rb +115 -24
- data/test/data/__eof2.rb +1 -0
- data/test/data/__eof5.rb +2 -0
- data/test/data/__eof6.rb +2 -0
- data/test/data/cvtesc.rb +17 -0
- data/test/data/g.rb +6 -0
- data/test/data/hd0.rb +3 -0
- data/test/data/hdateof.rb +2 -0
- data/test/data/hdempty.rb +3 -0
- data/test/data/hdr.rb +9 -0
- data/test/data/hdr_dos.rb +13 -0
- data/test/data/hdr_dos2.rb +18 -0
- data/test/data/heart.rb +2 -0
- data/test/data/here_escnl.rb +25 -0
- data/test/data/here_escnl_dos.rb +20 -0
- data/test/data/here_squote.rb +3 -0
- data/test/data/heremonsters.rb +140 -0
- data/test/data/heremonsters.rb.broken +68 -0
- data/test/data/heremonsters.rb.broken.save +68 -0
- data/test/data/heremonsters_dos.rb +140 -0
- data/test/data/heremonsters_dos.rb.broken +68 -0
- data/test/data/illegal_oneliners.rb +1 -0
- data/test/data/illegal_stanzas.rb +0 -0
- data/test/data/make_ws_strdelim.rb +22 -0
- data/test/data/maven2_builer_test.rb +82 -0
- data/test/data/migration.rb +8944 -0
- data/test/data/modl.rb +6 -0
- data/test/data/modl_dos.rb +7 -0
- data/test/data/modl_fails.rb +10 -0
- data/test/data/multilinestring.rb +6 -0
- data/test/data/oneliners.rb +555 -0
- data/test/data/p-op.rb +2 -0
- data/test/data/p.rb +3 -1710
- data/test/data/s.rb +90 -21
- data/test/data/simple.rb +1 -0
- data/test/data/simple_dos.rb +1 -0
- data/test/data/stanzas.rb +1194 -0
- data/test/data/strdelim_crlf.rb +6 -0
- data/test/data/stuff.rb +6 -0
- data/test/data/stuff2.rb +5 -0
- data/test/data/stuff3.rb +6 -0
- data/test/data/stuff4.rb +6 -0
- data/test/data/tkweird.rb +20 -0
- data/test/data/unending_stuff.rb +5 -0
- data/test/data/whatnot.rb +8 -0
- data/test/data/ws_strdelim.rb +0 -0
- data/test/test.sh +239 -0
- data/testing.txt +39 -50
- metadata +110 -12
- data/test/code/dl_all_gems.rb +0 -43
- data/test/code/unpack_all_gems.rb +0 -15
- data/test/data/gemlist.txt +0 -280
data/test/code/errscan
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'test/code/rubylexervsruby'
|
2
|
+
opts = GetoptLong.new( [ '--ends', '-e', GetoptLong::OPTIONAL_ARGUMENT] )
|
3
|
+
endcount=0
|
4
|
+
opts.each {|opt,arg|
|
5
|
+
if opt=='--ends'
|
6
|
+
if arg==''
|
7
|
+
endcount=1
|
8
|
+
else
|
9
|
+
endcount=arg.to_i
|
10
|
+
end
|
11
|
+
end
|
12
|
+
}
|
13
|
+
|
14
|
+
ends="end\n"*endcount
|
15
|
+
|
16
|
+
endcode=ends#+"__END__\n"
|
17
|
+
|
18
|
+
lines=IO.readlines(ARGV.first)
|
19
|
+
|
20
|
+
low=1
|
21
|
+
high=lines.size-1
|
22
|
+
|
23
|
+
class DontEvalThis<Exception; end
|
24
|
+
|
25
|
+
def syntax_ok?(code)
|
26
|
+
caught=
|
27
|
+
begin
|
28
|
+
catch :dontevalthis do
|
29
|
+
eval "
|
30
|
+
BEGIN{throw :dontevalthis, :notevaled}
|
31
|
+
BEGIN{raise DontEvalThis}
|
32
|
+
"+code
|
33
|
+
end
|
34
|
+
rescue DontEvalThis:
|
35
|
+
puts "first level eval barrier failed!!!"
|
36
|
+
caught=:notevaled
|
37
|
+
rescue Exception:
|
38
|
+
caught=:unexpected
|
39
|
+
end
|
40
|
+
case caught
|
41
|
+
when :unexpected
|
42
|
+
return false
|
43
|
+
when :notevaled
|
44
|
+
return true
|
45
|
+
else #wtf?
|
46
|
+
puts "eval barriers totally bypassed?!?!?"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
span=9999999999999999999999999999
|
51
|
+
loop do
|
52
|
+
break if high-low>=span
|
53
|
+
span=high-low
|
54
|
+
adjust=1
|
55
|
+
mid=(low+high)/2
|
56
|
+
begin
|
57
|
+
text=lines.dup
|
58
|
+
realmid=mid+adjust
|
59
|
+
realmid=[[realmid,low].max,high].min
|
60
|
+
text=text[0...realmid] << endcode
|
61
|
+
text=text.to_s
|
62
|
+
adjust*=-2
|
63
|
+
end until syntax_ok?text
|
64
|
+
|
65
|
+
if RubyLexerVsRuby.rubylexervsruby(ARGV.first+".chunk#{low..high}", text)
|
66
|
+
low=realmid
|
67
|
+
else
|
68
|
+
high=realmid
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
p low..high
|
data/test/code/lexloop
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
ruby -Ilib test/code/tokentest.rb --loop --maxws \
|
2
|
+
/home/caleb/sandbox/rubyparser/babyparser.rb \
|
3
|
+
/home/caleb/sandbox/rubyparser/hd2.rb \
|
4
|
+
/home/caleb/sandbox/rubyparser/test_rubyparser.rb \
|
5
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/rulexer.rb \
|
6
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/tokenprinter.rb \
|
7
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/0.6.2.rb \
|
8
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/0.7.1.rb \
|
9
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/charset.rb \
|
10
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/symboltable.rb \
|
11
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/charhandler.rb \
|
12
|
+
/home/caleb/sandbox/rubylexer/lib/rubylexer/0.7.0.rb \
|
13
|
+
/home/caleb/sandbox/rubylexer/lib/assert.rb \
|
14
|
+
>/dev/null 2>&1
|
data/test/code/locatetest.rb
CHANGED
@@ -1,6 +1,30 @@
|
|
1
|
+
=begin legal crap
|
2
|
+
rubylexer - a ruby lexer written in ruby
|
3
|
+
Copyright (C) 2004,2005,2008 Caleb Clausen
|
4
|
+
|
5
|
+
This library is free software; you can redistribute it and/or
|
6
|
+
modify it under the terms of the GNU Lesser General Public
|
7
|
+
License as published by the Free Software Foundation; either
|
8
|
+
version 2.1 of the License, or (at your option) any later version.
|
9
|
+
|
10
|
+
This library is distributed in the hope that it will be useful,
|
11
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
+
Lesser General Public License for more details.
|
14
|
+
|
15
|
+
You should have received a copy of the GNU Lesser General Public
|
16
|
+
License along with this library; if not, write to the Free Software
|
17
|
+
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
18
|
+
=end
|
19
|
+
|
1
20
|
require 'test/code/rubylexervsruby'
|
21
|
+
require 'test/code/strgen'
|
22
|
+
|
23
|
+
class RubyLexer
|
24
|
+
module LocateTest
|
2
25
|
#ENV['RUBY']||='ruby'
|
3
26
|
$RUBY=ENV['RUBY']||'ruby'
|
27
|
+
skip_til=ENV['SKIP_TIL']
|
4
28
|
#test $RUBY || export RUBY=ruby
|
5
29
|
|
6
30
|
#$RUBYLEXERVSRUBY="#$RUBY test/code/rubylexervsruby.rb"
|
@@ -31,19 +55,137 @@ ruby -e 'print ($:.sort.uniq+[""]).join"\n"'|xargs -i ls "{}/*.rb" >> test/resul
|
|
31
55
|
|
32
56
|
=end
|
33
57
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
]
|
58
|
+
bindirs=ENV['PATH'].split(':')+['/sbin','/usr/sbin']
|
59
|
+
|
60
|
+
|
61
|
+
puts "hack hack hack: absolute path in ruby test file list"
|
62
|
+
RUBYBINS=
|
63
|
+
bindirs.map{|dir| Dir[dir+"/*"].select{|prog|
|
64
|
+
line1=File.open(prog){|f| f.readline} rescue next
|
65
|
+
%r{\A#!([^\s]*/env )?[^\s]*/j?ruby(?:1\.[6789])(?:\s|\n)}===line1
|
66
|
+
}}.flatten
|
67
|
+
TEST_THESE_FIRST=
|
68
|
+
%w[
|
69
|
+
|
70
|
+
|
71
|
+
|
72
|
+
|
73
|
+
|
74
|
+
|
75
|
+
|
76
|
+
|
77
|
+
|
78
|
+
|
79
|
+
|
80
|
+
|
81
|
+
/home/caleb/sandbox/rubylexer/jewels/xrefresh-server-0.1.0/lib/xrefresh-server.rb
|
82
|
+
/home/caleb/sandbox/rubylexer/jewels/shattered-0.7.0/test/acceptance/lib/mesh_test.rb
|
83
|
+
/home/caleb/sandbox/rubylexer/jewels/rutils-0.2.3/lib/countries/countries.rb
|
84
|
+
/home/caleb/sandbox/rubylexer/jewels/ruport-1.6.1/examples/trac_ticket_status.rb
|
85
|
+
/home/caleb/sandbox/rubylexer/jewels/rubysdl-2.0.1/extconf.rb
|
86
|
+
/home/caleb/sandbox/rubylexer/jewels/rubysdl-1.1.0/extconf.rb
|
87
|
+
/home/caleb/sandbox/rubylexer/jewels/ruby-msg-1.3.1/lib/msg/rtf.rb
|
88
|
+
/home/caleb/sandbox/rubylexer/jewels/ruby-mediawiki-0.1/apps/iso_639_leecher.rb
|
89
|
+
/home/caleb/sandbox/rubylexer/jewels/ruby-finance-0.2.2/lib/finance/quote/yahoo/australia.rb
|
90
|
+
/home/caleb/sandbox/rubylexer/jewels/roby-0.7.2/test/test_task.rb
|
91
|
+
/home/caleb/sandbox/rubylexer/jewels/reve-0.0.94/test/test_reve.rb
|
92
|
+
/home/caleb/sandbox/rubylexer/jewels/remote_api-0.2.1/test/spec_test.rb
|
93
|
+
/home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/getartist.rb
|
94
|
+
/home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/getlabel.rb
|
95
|
+
/home/caleb/sandbox/rubylexer/jewels/rbrainz-0.4.1/examples/gettrack.rb
|
96
|
+
/home/caleb/sandbox/rubylexer/jewels/railscart-0.0.4/starter_app/vendor/plugins/engines/init.rb
|
97
|
+
/home/caleb/sandbox/rubylexer/jewels/ok-extensions-1.0.15/test/extensions/test_object.rb
|
98
|
+
/home/caleb/sandbox/rubylexer/jewels/oai-0.0.8/lib/oai/harvester/logging.rb
|
99
|
+
/home/caleb/sandbox/rubylexer/jewels/oai-0.0.8/examples/providers/dublin_core.rb
|
100
|
+
/home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/assert-text-equal.rbc
|
101
|
+
/home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-engines.rbc
|
102
|
+
/home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-erubis.rbc
|
103
|
+
/home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test-users-guide.rbc
|
104
|
+
/home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/test.rbc
|
105
|
+
/home/caleb/sandbox/rubylexer/jewels/erubis-2.6.2/test/testutil.rbc
|
106
|
+
/home/caleb/sandbox/rubylexer/jewels/fb-0.5.5/test/CursorTestCases.rb
|
107
|
+
/home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/examples/auth.rb
|
108
|
+
/home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/examples/upload.rb
|
109
|
+
/home/caleb/sandbox/rubylexer/jewels/flickraw-0.4.5/test/test.rb
|
110
|
+
/home/caleb/sandbox/rubylexer/jewels/fox-tool-0.10.0-preview/fox-tool/examples/input.rbin
|
111
|
+
/home/caleb/sandbox/rubylexer/jewels/fox-tool-0.10.0-preview/fox-tool/examples/cvs/Base/print.rbin
|
112
|
+
/home/caleb/sandbox/rubylexer/jewels/foxGUIb_1.0.0/foxguib_1.0.0/foxguib/src/gui/_guib_genruby.rbin
|
113
|
+
/home/caleb/sandbox/rubylexer/jewels/hpricot_scrub-0.3.2/test/scrubber_data.rb
|
114
|
+
/home/caleb/sandbox/rubylexer/jewels/htmltools-1.10/test/tc_stacking-parser.rb
|
115
|
+
/home/caleb/sandbox/rubylexer/jewels/ludy-0.1.13/test/deprecated/ts_ludy.rb
|
116
|
+
/home/caleb/sandbox/rubylexer/jewels/menu_helper-0.0.5/test/unit/menu_test.rb
|
117
|
+
/home/caleb/sandbox/rubylexer/jewels/mod_spox-0.0.5/data/mod_spox/extras/PhpCli.rb
|
118
|
+
/home/caleb/sandbox/rubylexer/jewels/motiro-0.6.11/app/core/wiki_page_not_found.rb
|
119
|
+
/home/caleb/sandbox/rubylexer/jewels/motiro-0.6.11/vendor/plugins/globalize/generators/globalize/templates/migration.rb.gz
|
120
|
+
|
121
|
+
]+[
|
122
|
+
"/home/caleb/sandbox/rubylexer/jewels/core_ex-0.6.6.3/lib/core_ex/numeric.rb",
|
123
|
+
"/home/caleb/sandbox/rubylexer/jewels/cerberus-0.3.6/test/bjam_builder_test.rb",
|
124
|
+
"/home/caleb/sandbox/rubylexer/jewels/cerberus-0.3.6/test/maven2_builer_test.rb",
|
125
|
+
"/home/caleb/sandbox/rubylexer/jewels/buildr-1.3.1.1/lib/buildr/java/groovyc.rb",
|
126
|
+
"/home/caleb/sandbox/rubylexer/jewels/adhearsion-0.7.7/apps/default/helpers/micromenus.rb",
|
127
|
+
"/home/caleb/rubies/ruby-1.8.7/instruby.rb",
|
128
|
+
"/home/caleb/sandbox/rubylexer/jewels/RuCodeGen-0.3.1/lib/rucodegen/value_incapsulator.rb",
|
129
|
+
|
130
|
+
"/home/caleb/sandbox/rubylexer/jewels/Wiki2Go-1.17.3/test/TestWiki2GoServlet.rb",
|
131
|
+
"/home/caleb/rubies/ruby-1.8.7/test/rss/test_parser_atom_entry.rb",
|
132
|
+
"/home/caleb/sandbox/rubylexer/jewels/active_form-0.0.8/test/elements/test_base_element.rb",
|
133
|
+
"/home/caleb/sandbox/rubylexer/jewels/dohruby-0.2.1/bin/create_database.rb",
|
134
|
+
"/home/caleb/sandbox/rubylexer/jewels/depager-0.2.2/examples/c89/c89.tab.rb",
|
135
|
+
|
136
|
+
"/home/caleb/sandbox/rubylexer/jewels/samizdat-0.6.1/samizdat/lib/samizdat/storage.rb",
|
137
|
+
"/home/caleb/sandbox/rubylexer/jewels/math3d-0.04/tests/make_tests.rb",
|
138
|
+
"/home/caleb/sandbox/rubylexer/jewels/QuickBaseClient.rb/quickbasecontactsappbuilder.rb",
|
139
|
+
"/home/caleb/sandbox/rubylexer/jewels/QuickBaseClient.rb/quickbaseclient.rb",
|
140
|
+
"/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._000-IVYTranslater.rb",
|
141
|
+
"/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._002-ApplicationList.rb",
|
142
|
+
"/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/examples/._001-UnBind.rb",
|
143
|
+
"/home/caleb/sandbox/rubylexer/jewels/ruby-ivy_0.1.0/ruby-ivy/._extconf.rb",
|
144
|
+
"/home/caleb/sandbox/rubylexer/jewels/smf-0.15.10/sample/virtual-samp.rb",
|
145
|
+
"/home/caleb/sandbox/rubylexer/jewels/syntax/syntax.rb",
|
146
|
+
"/home/caleb/sandbox/rubylexer/jewels/rex-1.0rc1/rex/packages/rex/test/rex-20060511.rb",
|
147
|
+
"/home/caleb/sandbox/rubylexer/jewels/rex-1.0rc1/rex/packages/rex/test/rex-20060125.rb",
|
148
|
+
# "/home/caleb/sandbox/rubylexer/jewels/japanese-zipcodes-0.0.20080227/lib/japanese/zipcodes.rb", #huge!!!!!
|
149
|
+
]
|
150
|
+
RUBYLIST=TEST_THESE_FIRST+RUBYBINS+
|
151
|
+
[RLROOT+"/test/data/p.rb",
|
152
|
+
*Dir["test/data/*.rb"]+
|
153
|
+
`(
|
154
|
+
locate /inline.rb /tk.rb;
|
155
|
+
locate examples/examples_test.rb;
|
156
|
+
locate .rb;
|
157
|
+
locate rakefile; locate Rakefile; locate RAKEFILE;
|
158
|
+
)|egrep -v '(/test/(results|data)/|.broken$)'`.
|
159
|
+
split("\n").reject{|i| %r(japanese/zipcodes\.rb\Z)===i }
|
160
|
+
]
|
161
|
+
RUBYLIST.uniq!
|
162
|
+
if skip_til
|
163
|
+
skip_til=RUBYLIST.index(skip_til)
|
164
|
+
skip_til or fail "SKIP_TIL not found in list of rubies"
|
165
|
+
RUBYLIST.slice! 0...skip_til
|
166
|
+
end
|
167
|
+
def self.main
|
168
|
+
for i in RUBYLIST do
|
40
169
|
# system $RUBYLEXERVSRUBY, i
|
41
170
|
#hmm, rubylexervsruby needs to be upgraded to not regard an output
|
42
171
|
#consisting entirely of warnings as a failure.
|
43
172
|
#if no 'warning' (in any capitalization) for 4 or more lines
|
44
|
-
|
173
|
+
begin
|
174
|
+
# puts File.readlines("/proc/#{$$}/status").grep(/^VmSize:\s/)
|
175
|
+
File.exist? i or next
|
176
|
+
RubyLexerVsRuby.rubylexervsruby i,nil,nil,true #or puts "syntax error in #{i}"
|
177
|
+
rescue Interrupt; raise
|
178
|
+
rescue Exception=>e
|
179
|
+
puts "error in: "+i
|
180
|
+
puts e
|
181
|
+
puts e.backtrace.map{|s| " from: "+s}.join("\n")
|
182
|
+
end
|
183
|
+
end
|
45
184
|
end
|
46
|
-
|
47
185
|
#for i in test/data/p.rb `(locate /tk.rb;locate examples/examples_test.rb;#locate .rb; locate rakefile; locate Rakefile; locate RAKEFILE)|egrep -v '/#test/results/'`; do
|
48
186
|
# $RUBYLEXERVSRUBY $i
|
49
187
|
#done
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
RubyLexer::LocateTest.main if $0==__FILE__
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'open3'
|
2
|
+
|
3
|
+
require 'test/code/testcases'
|
4
|
+
require 'test/code/rubylexervsruby'
|
5
|
+
SEP='';
|
6
|
+
'caleb clausen'.each_byte{|ch| SEP<<ch.to_s(2).gsub('0','-').gsub('1','+')}
|
7
|
+
SEP<<'(0)'
|
8
|
+
|
9
|
+
#require 'test/code/rubyoracle'
|
10
|
+
class<<RubyLexerVsRuby
|
11
|
+
|
12
|
+
def progress ruby,input; end
|
13
|
+
=begin oracular version was just a bad idea....
|
14
|
+
def ruby_parsedump(input,output,ruby)
|
15
|
+
#todo: use ruby's md5 lib
|
16
|
+
#recursive ruby call here is unavoidable because -y flag has to be set
|
17
|
+
|
18
|
+
@oracle||= Open3.popen3("#{ruby} -w -y -c")
|
19
|
+
@oracle[0].write IO.read(input)
|
20
|
+
@oracle[0].flush
|
21
|
+
#timeout=Time.now+0.1
|
22
|
+
data=''
|
23
|
+
while data.empty? # and Time.now<timeout
|
24
|
+
begin
|
25
|
+
data<< chunk=@oracle[2].read_nonblock(1024)
|
26
|
+
timeout=Time.now+0.02
|
27
|
+
rescue EOFError
|
28
|
+
# data<<"\nError: premature eof...\n"
|
29
|
+
break
|
30
|
+
rescue Errno::EAGAIN
|
31
|
+
break if data[/^Reading a token: \Z/] and Time.now>=timeout
|
32
|
+
end while chunk
|
33
|
+
end
|
34
|
+
|
35
|
+
status=0
|
36
|
+
lines=data.split("\n")
|
37
|
+
File.open(output,"w") { |outfd|
|
38
|
+
lines.each{|line|
|
39
|
+
outfd.puts(line) if /^Shifting/===line
|
40
|
+
if /^#{DeleteWarns::WARNERRREX}|^Error|^(Now at end of input\.)/o===line
|
41
|
+
outfd.puts(line)
|
42
|
+
if status.zero? and $2!="warning"
|
43
|
+
status=1 unless $4 #unless end of input seen
|
44
|
+
@oracle.each{|fd| fd.close} if @oracle
|
45
|
+
@oracle=nil
|
46
|
+
end
|
47
|
+
end
|
48
|
+
}
|
49
|
+
}
|
50
|
+
return status
|
51
|
+
end
|
52
|
+
=end
|
53
|
+
end
|
54
|
+
|
55
|
+
if __FILE__==$0
|
56
|
+
require 'test/unit'
|
57
|
+
|
58
|
+
class LexerTests<Test::Unit::TestCase
|
59
|
+
class LexerTestFailure<RuntimeError; end
|
60
|
+
class DifferencesFromMRILex<LexerTestFailure; end
|
61
|
+
|
62
|
+
i=-1
|
63
|
+
test_code= TestCases::TESTCASES.map{|tc|
|
64
|
+
i+=1
|
65
|
+
name="testcase_#{i}__"
|
66
|
+
esctc=tc.gsub(/['\\]/){"\\"+$&}
|
67
|
+
%[
|
68
|
+
define_method '#{name}' do
|
69
|
+
difflines=[]
|
70
|
+
begin
|
71
|
+
res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines)
|
72
|
+
difflines.empty? or raise DifferencesFromMRILex, difflines
|
73
|
+
res or raise LexerTestFailure, ''
|
74
|
+
rescue Interrupt: exit
|
75
|
+
rescue Exception=>e:
|
76
|
+
e.message<<"\n"+'while lexing: #{esctc}'
|
77
|
+
raise e
|
78
|
+
end
|
79
|
+
end
|
80
|
+
]
|
81
|
+
}.to_s
|
82
|
+
|
83
|
+
illegal_test_code= TestCases::ILLEGAL_TESTCASES.map{|tc|
|
84
|
+
i+=1
|
85
|
+
name="testcase_#{i}__"
|
86
|
+
esctc=tc.gsub(/['\\]/){"\\"+$&}
|
87
|
+
%[
|
88
|
+
define_method '#{name}' do
|
89
|
+
difflines=[]
|
90
|
+
begin
|
91
|
+
res=RubyLexerVsRuby.rubylexervsruby('__#{name}','#{esctc}',difflines)
|
92
|
+
difflines.empty? or raise DifferencesFromMRILex, difflines
|
93
|
+
res or raise LexerTestFailure, ''
|
94
|
+
rescue LexerTestFailure:
|
95
|
+
puts 'warning: test failure lexing "#{esctc}"'
|
96
|
+
rescue Interrupt: exit
|
97
|
+
rescue Exception=>e:
|
98
|
+
e.message<<"\n"+'while lexing: #{esctc}'
|
99
|
+
raise e
|
100
|
+
end
|
101
|
+
end
|
102
|
+
]
|
103
|
+
}.to_s
|
104
|
+
|
105
|
+
|
106
|
+
# puts test_code+illegal_test_code
|
107
|
+
eval test_code+illegal_test_code
|
108
|
+
end
|
109
|
+
end
|
@@ -1,4 +1,22 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
|
+
=begin legal crap
|
3
|
+
rubylexer - a ruby lexer written in ruby
|
4
|
+
Copyright (C) 2004,2005,2008 Caleb Clausen
|
5
|
+
|
6
|
+
This library is free software; you can redistribute it and/or
|
7
|
+
modify it under the terms of the GNU Lesser General Public
|
8
|
+
License as published by the Free Software Foundation; either
|
9
|
+
version 2.1 of the License, or (at your option) any later version.
|
10
|
+
|
11
|
+
This library is distributed in the hope that it will be useful,
|
12
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14
|
+
Lesser General Public License for more details.
|
15
|
+
|
16
|
+
You should have received a copy of the GNU Lesser General Public
|
17
|
+
License along with this library; if not, write to the Free Software
|
18
|
+
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
19
|
+
=end
|
2
20
|
#$DEBUG=$VERBOSE=true
|
3
21
|
$Debug=true
|
4
22
|
require "getoptlong"
|
@@ -39,15 +57,20 @@ def ruby_parsedump(input,output,ruby)
|
|
39
57
|
}
|
40
58
|
}
|
41
59
|
ENABLEMD5 and status==0 and system "md5sum #{input} > #{input}.md5" #compute sum only if no errors
|
60
|
+
return status>>8
|
42
61
|
end
|
43
62
|
|
44
63
|
def head(fname)
|
45
|
-
print "foobaaaaaaaaar\n\n\n\n\n\n"
|
46
64
|
File.open(fname){|fd| print(fd.read(512)+"\n") }
|
47
65
|
end
|
48
66
|
|
49
|
-
def rubylexervsruby(input,stringdata=nil,&ignore_it)
|
50
67
|
|
68
|
+
def progress ruby,input
|
69
|
+
print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
|
70
|
+
$stdout.flush
|
71
|
+
end
|
72
|
+
|
73
|
+
def rubylexervsruby(input,stringdata=nil,difflines=nil,bulk=nil,&ignore_it)
|
51
74
|
#cmdpath= `which #$0`
|
52
75
|
cmddir=Dir.getwd+"/test/code/"
|
53
76
|
base='test/results/'+File.basename(input)
|
@@ -67,12 +90,14 @@ expected_failures=Dir.getwd+"/test/code/"+File.basename(input)+".expected_failur
|
|
67
90
|
|
68
91
|
nop_ruby "#{input[/\.gz$/]&&'z'}cat", input, nopfile, stringdata
|
69
92
|
|
70
|
-
print "executing: #{ruby} -Ilib test/code/tokentest.rb --keepws #{input}\n"
|
71
|
-
|
72
|
-
ruby_parsedump nopfile, origfile, ruby
|
73
|
-
|
74
93
|
|
94
|
+
legal=ruby_parsedump nopfile, origfile, ruby
|
95
|
+
if false&&legal.nonzero?
|
96
|
+
puts "skipping #{input}; not legal"
|
97
|
+
return
|
98
|
+
end
|
75
99
|
|
100
|
+
progress ruby, input
|
76
101
|
tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new, nil, _ttfile
|
77
102
|
tokentest nopfile, RubyLexer, RubyLexer::KeepWsTokenPrinter.new(' '), nil, mttfile
|
78
103
|
|
@@ -81,7 +106,7 @@ ruby_parsedump _ttfile, p_ttfile, ruby
|
|
81
106
|
ruby_parsedump mttfile, pmttfile, ruby
|
82
107
|
|
83
108
|
if File.exists?(p_ttfile)
|
84
|
-
IO.popen("diff
|
109
|
+
IO.popen("diff --unified=1 -b #{origfile} #{p_ttfile}"){ |pipe|
|
85
110
|
File.open(p_ttdiff,"w") { |diff|
|
86
111
|
DeleteWarns.deletewarns(pipe){|s| diff.print s}
|
87
112
|
}
|
@@ -90,7 +115,7 @@ if File.exists?(p_ttfile)
|
|
90
115
|
end
|
91
116
|
|
92
117
|
if File.exists?(pmttfile)
|
93
|
-
IO.popen("diff
|
118
|
+
IO.popen("diff --unified=1 -b #{origfile} #{pmttfile}"){ |pipe|
|
94
119
|
File.open(pmttdiff,"w") { |diff|
|
95
120
|
DeleteWarns.deletewarns(pipe){|s| diff.print s}
|
96
121
|
}
|
@@ -101,10 +126,11 @@ end
|
|
101
126
|
list=[]
|
102
127
|
#nonwarn4=/(^(?![^\n]*warning[^\n]*)[^\n]*\n){4,}/im
|
103
128
|
#4 or more non-warning lines:
|
104
|
-
nonwarn4=/^(?:(?![^\r\n]*warning)[^\
|
129
|
+
nonwarn4=/^(?:(?![^\r\n]*warning)[^\n]+\n){4,}/mi
|
105
130
|
result=true
|
131
|
+
(system "ruby -c #{input} >/dev/null 2>&1" or expected="(expected) ") unless stringdata
|
106
132
|
for name in [p_ttdiff,pmttdiff] do
|
107
|
-
i=File.read(name)
|
133
|
+
i=File.read(name) rescue next
|
108
134
|
# i.tr("\r","\n")
|
109
135
|
# i.gsub!(/^\n/m, '')
|
110
136
|
i.sub!(/\A([^\r\n]+(\r\n?|\n\r?)){2}/, '') #remove 1st 2 lines
|
@@ -116,9 +142,10 @@ for name in [p_ttdiff,pmttdiff] do
|
|
116
142
|
|
117
143
|
unless list.empty?
|
118
144
|
list=list.join("\n") +"\n"
|
119
|
-
unless (File.exists?(expected_failures) and File.read(expected_failures)
|
120
|
-
|
121
|
-
|
145
|
+
unless (File.exists?(expected_failures) and File.read(expected_failures)==list)
|
146
|
+
result=!!expected
|
147
|
+
print "#{expected}error in: #{input}\n"
|
148
|
+
(difflines ? difflines.push(*list) : print(list)) unless expected
|
122
149
|
end
|
123
150
|
list=[]
|
124
151
|
end
|
@@ -126,6 +153,8 @@ end
|
|
126
153
|
|
127
154
|
#print( list.join("\n") +"\n")
|
128
155
|
#Dir.chdir olddir
|
156
|
+
File.unlink(*Dir[base+".{tt,mtt,nop}{,.prs}{,.diff}"]) if result and bulk
|
157
|
+
|
129
158
|
return result
|
130
159
|
|
131
160
|
=begin
|
@@ -147,10 +176,14 @@ end
|
|
147
176
|
return false
|
148
177
|
=end
|
149
178
|
|
179
|
+
rescue Interrupt: raise
|
150
180
|
rescue Exception
|
151
|
-
|
181
|
+
File.exist? nopfile and
|
182
|
+
system "ruby -c #{nopfile} >/dev/null 2>&1" or expected="(expected) " unless stringdata
|
152
183
|
print "#{expected}error in: #{input}\n"
|
153
|
-
|
184
|
+
File.unlink(*Dir[base+".{tt,mtt,nop}{,.prs}{,.diff}"]) if expected and bulk
|
185
|
+
return true if expected
|
186
|
+
raise
|
154
187
|
end
|
155
188
|
end
|
156
189
|
|
@@ -165,6 +198,11 @@ opts.each{|opt,arg|
|
|
165
198
|
}
|
166
199
|
|
167
200
|
input||=ARGV[0]
|
201
|
+
|
202
|
+
unless input
|
203
|
+
input="-"
|
204
|
+
stringdata=$stdin.read
|
205
|
+
end
|
168
206
|
RubyLexerVsRuby.rubylexervsruby(input,stringdata) and exit 0
|
169
207
|
|
170
208
|
exit 1
|