rubylexer 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/History.txt +90 -0
  2. data/Manifest.txt +54 -3
  3. data/README.txt +4 -7
  4. data/Rakefile +3 -2
  5. data/lib/rubylexer.rb +856 -323
  6. data/lib/rubylexer/0.7.0.rb +11 -2
  7. data/lib/rubylexer/0.7.1.rb +2 -0
  8. data/lib/rubylexer/charhandler.rb +4 -4
  9. data/lib/rubylexer/context.rb +86 -9
  10. data/lib/rubylexer/rulexer.rb +455 -101
  11. data/lib/rubylexer/token.rb +166 -43
  12. data/lib/rubylexer/tokenprinter.rb +16 -8
  13. data/lib/rubylexer/version.rb +1 -1
  14. data/rubylexer.vpj +98 -0
  15. data/test/code/all_the_gems.rb +33 -0
  16. data/test/code/all_the_raas.rb +226 -0
  17. data/test/code/all_the_rubies.rb +2 -0
  18. data/test/code/deletewarns.rb +19 -1
  19. data/test/code/dumptokens.rb +39 -8
  20. data/test/code/errscan +2 -0
  21. data/test/code/isolate_error.rb +72 -0
  22. data/test/code/lexloop +14 -0
  23. data/test/code/locatetest.rb +150 -8
  24. data/test/code/regression.rb +109 -0
  25. data/test/code/rubylexervsruby.rb +53 -15
  26. data/test/code/strgen.rb +138 -0
  27. data/test/code/tarball.rb +144 -0
  28. data/test/code/testcases.rb +11 -0
  29. data/test/code/tokentest.rb +115 -24
  30. data/test/data/__eof2.rb +1 -0
  31. data/test/data/__eof5.rb +2 -0
  32. data/test/data/__eof6.rb +2 -0
  33. data/test/data/cvtesc.rb +17 -0
  34. data/test/data/g.rb +6 -0
  35. data/test/data/hd0.rb +3 -0
  36. data/test/data/hdateof.rb +2 -0
  37. data/test/data/hdempty.rb +3 -0
  38. data/test/data/hdr.rb +9 -0
  39. data/test/data/hdr_dos.rb +13 -0
  40. data/test/data/hdr_dos2.rb +18 -0
  41. data/test/data/heart.rb +2 -0
  42. data/test/data/here_escnl.rb +25 -0
  43. data/test/data/here_escnl_dos.rb +20 -0
  44. data/test/data/here_squote.rb +3 -0
  45. data/test/data/heremonsters.rb +140 -0
  46. data/test/data/heremonsters.rb.broken +68 -0
  47. data/test/data/heremonsters.rb.broken.save +68 -0
  48. data/test/data/heremonsters_dos.rb +140 -0
  49. data/test/data/heremonsters_dos.rb.broken +68 -0
  50. data/test/data/illegal_oneliners.rb +1 -0
  51. data/test/data/illegal_stanzas.rb +0 -0
  52. data/test/data/make_ws_strdelim.rb +22 -0
  53. data/test/data/maven2_builer_test.rb +82 -0
  54. data/test/data/migration.rb +8944 -0
  55. data/test/data/modl.rb +6 -0
  56. data/test/data/modl_dos.rb +7 -0
  57. data/test/data/modl_fails.rb +10 -0
  58. data/test/data/multilinestring.rb +6 -0
  59. data/test/data/oneliners.rb +555 -0
  60. data/test/data/p-op.rb +2 -0
  61. data/test/data/p.rb +3 -1710
  62. data/test/data/s.rb +90 -21
  63. data/test/data/simple.rb +1 -0
  64. data/test/data/simple_dos.rb +1 -0
  65. data/test/data/stanzas.rb +1194 -0
  66. data/test/data/strdelim_crlf.rb +6 -0
  67. data/test/data/stuff.rb +6 -0
  68. data/test/data/stuff2.rb +5 -0
  69. data/test/data/stuff3.rb +6 -0
  70. data/test/data/stuff4.rb +6 -0
  71. data/test/data/tkweird.rb +20 -0
  72. data/test/data/unending_stuff.rb +5 -0
  73. data/test/data/whatnot.rb +8 -0
  74. data/test/data/ws_strdelim.rb +0 -0
  75. data/test/test.sh +239 -0
  76. data/testing.txt +39 -50
  77. metadata +110 -12
  78. data/test/code/dl_all_gems.rb +0 -43
  79. data/test/code/unpack_all_gems.rb +0 -15
  80. data/test/data/gemlist.txt +0 -280
@@ -1,3 +1,3 @@
1
1
  class RubyLexer
2
- VERSION='0.7.0'
2
+ VERSION='0.7.1'
3
3
  end
data/rubylexer.vpj ADDED
@@ -0,0 +1,98 @@
1
+ <!DOCTYPE Project SYSTEM "http://www.slickedit.com/dtd/vse/10.0/vpj.dtd">
2
+ <Project
3
+ Version="10.0"
4
+ VendorName="SlickEdit"
5
+ WorkingDir=".">
6
+ <Config
7
+ Name="Release"
8
+ OutputFile=""
9
+ CompilerConfigName="Latest Version">
10
+ <Menu>
11
+ <Target
12
+ Name="Compile"
13
+ MenuCaption="&amp;Compile"
14
+ CaptureOutputWith="ProcessBuffer"
15
+ SaveOption="SaveCurrent"
16
+ RunFromDir="%rw">
17
+ <Exec/>
18
+ </Target>
19
+ <Target
20
+ Name="Build"
21
+ MenuCaption="&amp;Build"
22
+ CaptureOutputWith="ProcessBuffer"
23
+ SaveOption="SaveWorkspaceFiles"
24
+ RunFromDir="%rw">
25
+ <Exec/>
26
+ </Target>
27
+ <Target
28
+ Name="Rebuild"
29
+ MenuCaption="&amp;Rebuild"
30
+ CaptureOutputWith="ProcessBuffer"
31
+ SaveOption="SaveWorkspaceFiles"
32
+ RunFromDir="%rw">
33
+ <Exec/>
34
+ </Target>
35
+ <Target
36
+ Name="Debug"
37
+ MenuCaption="&amp;Debug"
38
+ SaveOption="SaveNone"
39
+ RunFromDir="%rw">
40
+ <Exec/>
41
+ </Target>
42
+ <Target
43
+ Name="Execute"
44
+ MenuCaption="E&amp;xecute"
45
+ SaveOption="SaveNone"
46
+ RunFromDir="%rw">
47
+ <Exec CmdLine='".exe"'/>
48
+ </Target>
49
+ </Menu>
50
+ </Config>
51
+ <Files>
52
+ <Folder
53
+ Name="Source Files"
54
+ Filters="*.c;*.C;*.cc;*.cpp;*.cp;*.cxx;*.prg;*.pas;*.dpr;*.asm;*.s;*.bas;*.java;*.cs;*.sc;*.e;*.cob;*.html;*.rc;*.tcl;*.py;*.pl;*.rb">
55
+ <F N="lib/rubylexer/0.6.2.rb"/>
56
+ <F N="lib/rubylexer/0.6.rb"/>
57
+ <F N="lib/rubylexer/0.7.0.rb"/>
58
+ <F N="lib/rubylexer/0.7.1.rb"/>
59
+ <F N="test/code/all_the_gems.rb"/>
60
+ <F N="test/code/all_the_raas.rb"/>
61
+ <F N="test/code/all_the_rubies.rb"/>
62
+ <F N="lib/assert.rb"/>
63
+ <F N="lib/rubylexer/charhandler.rb"/>
64
+ <F N="lib/rubylexer/charset.rb"/>
65
+ <F N="lib/rubylexer/context.rb"/>
66
+ <F N="test/code/deletewarns.rb"/>
67
+ <F N="test/code/dumptokens.rb"/>
68
+ <F N="test/code/isolate_error.rb"/>
69
+ <F N="test/code/locatetest.rb"/>
70
+ <F N="test/code/regression.rb"/>
71
+ <F N="lib/rubylexer/rubycode.rb"/>
72
+ <F N="lib/rubylexer.rb"/>
73
+ <F N="test/code/rubylexervsruby.rb"/>
74
+ <F N="lib/rubylexer/rulexer.rb"/>
75
+ <F N="lib/rubylexer/symboltable.rb"/>
76
+ <F N="test/code/tarball.rb"/>
77
+ <F N="test/code/testcases.rb"/>
78
+ <F N="lib/rubylexer/token.rb"/>
79
+ <F N="lib/rubylexer/tokenprinter.rb"/>
80
+ <F N="test/code/tokentest.rb"/>
81
+ <F N="lib/rubylexer/version.rb"/>
82
+ </Folder>
83
+ <Folder
84
+ Name="Header Files"
85
+ Filters="*.h;*.H;*.hh;*.hpp;*.hxx;*.inc;*.sh;*.cpy;*.if"/>
86
+ <Folder
87
+ Name="Resource Files"
88
+ Filters="*.ico;*.cur;*.dlg"/>
89
+ <Folder
90
+ Name="Bitmaps"
91
+ Filters="*.bmp"/>
92
+ <Folder
93
+ Name="Other Files"
94
+ Filters="">
95
+ <F N="testing.txt"/>
96
+ </Folder>
97
+ </Files>
98
+ </Project>
@@ -0,0 +1,33 @@
1
+ #http://gems.rubyforge.org/yaml
2
+ #http://gems.rubyforge.org/gems/#{name}-#{version}.gem
3
+
4
+ require 'rubygems'
5
+ require 'yaml'
6
+ require 'open-uri'
7
+
8
+ require "test/code/tarball"
9
+
10
+ limit=(ENV['LIMIT']||20).to_i
11
+ offset=(ENV['OFFSET']||0).to_i
12
+
13
+ specs=open(ARGV.first||"http://gems.rubyforge.org/yaml"){|net| YAML.load net }
14
+
15
+ name2vers={}
16
+ specs.each{|bogus,spec|
17
+ name2vers[spec.name]||=[]
18
+ name2vers[spec.name]<<spec.version
19
+ }
20
+ specs=nil
21
+
22
+ name2vers.each_key{|name|
23
+ name2vers[name]=name2vers[name].max
24
+ }
25
+ #name2vers=name2vers.to_a[limit,offset]
26
+
27
+ name2vers.each{|name,version|
28
+ begin
29
+ Tarball.dl_and_unpack("jewels/","http://gems.rubyforge.org/gems/#{name}-#{version}.gem")
30
+ rescue Interrupt: exit
31
+ rescue Exception: #do nothing
32
+ end
33
+ }
@@ -0,0 +1,226 @@
1
+ require 'rubygems'
2
+ require 'hpricot'
3
+ require 'open-uri'
4
+ require 'algorithm/diff'
5
+ require 'pathname'
6
+
7
+ #require "test/code/all_the_gems"
8
+ require "test/code/tarball"
9
+
10
+ def fetch_with_cache(base,path,cachedir)
11
+
12
+ path[-1]==?/ and path+='index.html'
13
+
14
+ #make sure right dir structure for this path exists in cachedir
15
+ dnames=Pathname.new(path)
16
+ dirs=[]
17
+ while dnames.to_s["/"]
18
+ dnames=dnames.dirname
19
+ dirs.unshift cachedir+dnames.to_s
20
+ end
21
+ dirs.each{|dir| Dir.mkdir dir rescue nil}
22
+
23
+ #find latest cached version of this file in cachedir
24
+ latest_fname=nil
25
+ latest_date=Time.mktime '1970'
26
+ Dir[cachedir+path+".*"].each{|fname|
27
+ mtime=File.mtime(fname)
28
+ if mtime>=latest_date
29
+ latest_date=mtime
30
+ latest_fname=fname
31
+ end
32
+ }
33
+
34
+ options={"User-Agent"=>"all_the_raas.rb"}
35
+
36
+ #extract etag from latest name
37
+ if latest_fname
38
+ latest_etag=latest_fname[%r{\A#{Regexp.quote cachedir+path}\.(.*)\Z},1]
39
+ options['If-None-Match']=latest_etag unless latest_etag==''
40
+ end
41
+
42
+ #refetch the file if it has changed, otherwise use cached copy
43
+ begin
44
+ open(base+path, options){|net|
45
+ latest_fname=cachedir+path+"."+(net.meta['etag']||'')
46
+ File.open(latest_fname,'w'){|f| f.write net.read }
47
+ }
48
+ #puts "fetched a fresh #{base+path}"
49
+ rescue OpenURI::HTTPError=>e
50
+ raise e unless e.io.status.first=='304' and /Not Modified/i===e.io.status.last
51
+ #puts "reusing latest #{latest_fname}"
52
+ end
53
+
54
+ return File.open latest_fname
55
+ end
56
+
57
+ def changeratio(s1,s2)
58
+ diffslen=s1.diff(s2).inject(0){|sum,(msg,pos,data)|
59
+ sum+data.size
60
+ }
61
+ return diffslen.to_f/(s1.size+s2.size)
62
+ end
63
+
64
+ def sameproject?(s1,s2)
65
+ s1.casecmp(s2).zero? or
66
+ s1.index(s2)==0 or
67
+ s2.index(s1)==0 or
68
+ changeratio(s1,s2)<0.5
69
+ end
70
+
71
+
72
+ if __FILE__==$0
73
+ base="http://raa.ruby-lang.org/"
74
+
75
+ offset=(ENV['OFFSET']||0).to_i
76
+ limit=(ENV['LIMIT']||20).to_i
77
+
78
+ cachedir="jewels/"
79
+
80
+ Dir.mkdir cachedir rescue nil
81
+
82
+ #fetch list of all projects from raa's all.html
83
+ all_raas=fetch_with_cache(base,"all.html",cachedir)
84
+ tree=Hpricot(all_raas)
85
+ tree/=:table
86
+ tree.search(:thead).remove
87
+ tree/=:tr
88
+ #tree=tree[offset,limit]
89
+ urls=tree.map{|row|
90
+ begin
91
+ row.search(:td).first.search(:a).first[:href]
92
+ rescue Exception=>e
93
+ puts "failure #{e} in row #{row}" #wank about it
94
+ nil
95
+ end
96
+ }
97
+
98
+ RUBYFORGE=%r{\Ahttp://(?:[a-z0-9_+-:]+\.)*rubyforge.org/}i
99
+
100
+ PROTOCOLS=%w[http https ftp]
101
+ EXTENSIONS=%w[tar zip rb tgz tbz2 tbz gem]
102
+ EXTRA_EXTENSIONS=%w[gz bz2 Z]
103
+ EXTRA_EXTENSIONS_REX="\\.(?:#{EXTRA_EXTENSIONS.join'|'})"
104
+ VERSIONTOO='' #was: "(?:[_-](.*))"
105
+ ENDINGS="\\.(?:#{EXTENSIONS.join('|')})(?:#{EXTRA_EXTENSIONS_REX})?"
106
+ TARBALL=%r<
107
+ \A(?:#{PROTOCOLS.join('|')})://
108
+ (?:[^/]+/)+
109
+ (.*)
110
+ #{VERSIONTOO}
111
+ #{ENDINGS}
112
+ \Z
113
+ >ixo
114
+
115
+ $rubyforge_urls=0
116
+
117
+ #pp urls
118
+ #crawl raa's individual page for each project, looking for dl link
119
+ urls.map!{|url|
120
+ begin
121
+ tree=Hpricot(fetch_with_cache(base,url,cachedir))
122
+ # if url
123
+ # url=base+url
124
+ # tree=Hpricot(open(url))
125
+ # end
126
+ project=tree.search(:title).inner_html[/\ARAA - (.*)\Z/,1]
127
+ tree/=:table
128
+ trs=tree/:tr
129
+ dl=trs.find{|tr|
130
+ !tr.search("th[text()^='Download']").empty?
131
+ }
132
+ newurl=dl.search('td/a').first
133
+ if newurl
134
+ url=newurl[:href]
135
+ %r{\A(#{PROTOCOLS.join('|')})://}io===url or url="http://"+url
136
+ if RUBYFORGE===url
137
+ unless TARBALL===url and not /\.gem\Z/===url
138
+ url=nil
139
+ $rubyforge_urls += 1
140
+ end
141
+ end
142
+ url.slice!(/\#.*\Z/) if url #trim off url section
143
+ [url,project]
144
+ else
145
+ puts "couldn't find td/a in #{base+url}"
146
+ [nil,nil]
147
+ end
148
+ rescue Interrupt=>e
149
+ raise if e.class==Interrupt #^c only, dammit! #$^$&%'n Timeout::Error
150
+ rescue Exception=>e
151
+ puts "error: #{e} during url #{url}"
152
+ #wank about it
153
+ [nil,nil]
154
+ end
155
+ }
156
+
157
+
158
+ #pp urls
159
+ #resolve list of dl urls into urls to 'tarballs' (which is meant to include zip, gem, etc)
160
+ #dl urls found on raa might be a direct link to a tarball, or point to a page that
161
+ #points to the tarball
162
+ urls.map!{|url,project|
163
+ if TARBALL===url
164
+ urlproject=$1
165
+ versionstart=urlproject.rindex(/[_-]/)
166
+ urlproject.slice! versionstart..-1 if versionstart
167
+ if sameproject?(urlproject,project)
168
+ url
169
+ else
170
+ puts "uh-oh, project #{project} not found in url #{url}. urlproject was #{urlproject}"
171
+ end
172
+
173
+ elsif url
174
+ href=nil
175
+ begin
176
+ tree=Hpricot(open(url))
177
+ tree/=:a
178
+ tarball_url=tree.find{|a|
179
+ href=a[:href] or next
180
+ unless %r[\A(?:#{PROTOCOLS.join('|')})://]o===href #relative url?
181
+ # url+="/" unless url[-1]==?/
182
+ href="/"+href unless href[0]==?/
183
+ href=url[%r{\A[^/]+//[^/]+}]+href
184
+ end
185
+ TARBALL===href
186
+ }
187
+ if tarball_url
188
+ urlproject=$1
189
+ versionstart=urlproject.rindex(/[_-]/)
190
+ urlproject.slice! versionstart..-1 if versionstart
191
+ if sameproject?(urlproject,project)
192
+ href
193
+ else
194
+ puts "uh-oh, project #{project} not found in url #{url}. urlproject was #{urlproject}"
195
+ end
196
+ end
197
+
198
+ rescue Interrupt=>e
199
+ raise if e.class==Interrupt
200
+ puts "error: #{e} during page scan of url #{url}"
201
+ rescue Exception=>e
202
+ puts "error: #{e} during page scan of url #{url}"
203
+ nil
204
+ end
205
+ end
206
+ }
207
+
208
+ /unzip ([^\s]+)[\s\n]/i===`unzip -v`
209
+ unzip="unzip"
210
+ unzip+=" -L" if $1[0..2].to_f>=5.5
211
+
212
+ pp urls
213
+ #for each tarball url actually found, dl the tarball
214
+ urls.compact.each{|url|
215
+ begin
216
+ Tarball.dl_and_unpack(cachedir,url)
217
+ rescue Interrupt=>e
218
+ raise if e.class==Interrupt
219
+ #else do nothing
220
+ rescue Exception
221
+ #do nothing
222
+ end
223
+ }
224
+
225
+ p [:$rubyforge_urls, $rubyforge_urls]
226
+ end
@@ -0,0 +1,2 @@
1
+ load 'all_the_gems'
2
+ load 'all_the_raas'
@@ -1,11 +1,29 @@
1
1
  #!/usr/bin/env ruby
2
+ =begin legal crap
3
+ rubylexer - a ruby lexer written in ruby
4
+ Copyright (C) 2004,2005,2008 Caleb Clausen
5
+
6
+ This library is free software; you can redistribute it and/or
7
+ modify it under the terms of the GNU Lesser General Public
8
+ License as published by the Free Software Foundation; either
9
+ version 2.1 of the License, or (at your option) any later version.
10
+
11
+ This library is distributed in the hope that it will be useful,
12
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14
+ Lesser General Public License for more details.
15
+
16
+ You should have received a copy of the GNU Lesser General Public
17
+ License along with this library; if not, write to the Free Software
18
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
+ =end
2
20
 
3
21
  module DeleteWarns
4
22
  FN='[^\n]+'
5
23
  DATETIME='\d+-\d\d?-\d\d? \d\d:\d\d:\d\d\.\d+ -?\d+'
6
24
  INDENTLINE='(?: [^\n]*\n)'
7
25
 
8
- WARNERRREX='(?:Reading a token: )?-:(\d+): (warning|(?:syntax )error)(?:: ([^\n]+))?'
26
+ WARNERRREX='(?:Reading a token: )?-:(\d+): (warning|(?:syntax )?error)(?:: ([^\n]+))?'
9
27
 
10
28
  RE=%r"(?#--- #{FN} #{DATETIME}
11
29
  \+\+\+ #{FN} #{DATETIME}
@@ -1,4 +1,23 @@
1
1
  #!/usr/bin/env ruby
2
+ =begin legal crap
3
+ rubylexer - a ruby lexer written in ruby
4
+ Copyright (C) 2004,2005,2008 Caleb Clausen
5
+
6
+ This library is free software; you can redistribute it and/or
7
+ modify it under the terms of the GNU Lesser General Public
8
+ License as published by the Free Software Foundation; either
9
+ version 2.1 of the License, or (at your option) any later version.
10
+
11
+ This library is distributed in the hope that it will be useful,
12
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14
+ Lesser General Public License for more details.
15
+
16
+ You should have received a copy of the GNU Lesser General Public
17
+ License along with this library; if not, write to the Free Software
18
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
+ =end
20
+
2
21
  $Debug=true
3
22
  require 'rubylexer'
4
23
  require 'getoptlong'
@@ -16,14 +35,21 @@ class Token
16
35
  end
17
36
  end
18
37
 
19
- file=nil
38
+ silent=file=nil
20
39
 
21
40
  #allow -e
22
- opts=GetoptLong.new(["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT])
41
+ opts=GetoptLong.new(
42
+ ["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT],
43
+ ["--silent", "-s", GetoptLong::NO_ARGUMENT]
44
+ )
23
45
  opts.each{|opt,arg|
24
- opt=='--eval' or raise :impossible
25
- file=arg
26
- name='-e'
46
+ case opt
47
+ when '--eval'
48
+ file=arg
49
+ name='-e'
50
+ when '--silent'
51
+ silent=true
52
+ end
27
53
  }
28
54
 
29
55
  #determine input file and its name if not already known
@@ -31,11 +57,16 @@ file||=if name=ARGV.first
31
57
  File.open(name)
32
58
  else
33
59
  name='-'
34
- $stdin
60
+ $stdin.read
35
61
  end
36
62
 
37
63
  lexer=RubyLexer.new(name, file)
38
- until RubyLexer::EoiToken===(tok=lexer.get1token)
39
- puts tok.strify
64
+ if silent
65
+ until RubyLexer::EoiToken===(tok=lexer.get1token)
66
+ end
67
+ else
68
+ until RubyLexer::EoiToken===(tok=lexer.get1token)
69
+ puts tok.strify
70
+ end
40
71
  end
41
72
  puts tok.strify #print eoi token