fabiokung-ruby_parser 2.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +44 -0
- data/History.txt +134 -0
- data/Manifest.txt +13 -0
- data/README.txt +86 -0
- data/Rakefile +155 -0
- data/bin/ruby_parse +88 -0
- data/lib/gauntlet_rubyparser.rb +120 -0
- data/lib/ruby_lexer.rb +1331 -0
- data/lib/ruby_parser.rb +5545 -0
- data/lib/ruby_parser.y +1790 -0
- data/lib/ruby_parser_extras.rb +1031 -0
- data/test/test_ruby_lexer.rb +1738 -0
- data/test/test_ruby_parser.rb +466 -0
- data/test/test_ruby_parser_extras.rb +177 -0
- metadata +100 -0
data/.autotest
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'autotest/restart'
|
4
|
+
|
5
|
+
Autotest.add_hook :initialize do |at|
|
6
|
+
at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
|
7
|
+
at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test:../../sexp_processor/dev/lib"
|
8
|
+
at.add_exception 'unit'
|
9
|
+
at.add_exception 'coverage'
|
10
|
+
at.add_exception '.diff'
|
11
|
+
at.add_exception 'coverage.info'
|
12
|
+
|
13
|
+
at.unit_diff = "unit_diff -u -b"
|
14
|
+
|
15
|
+
at.add_mapping(/^lib\/.*\.y$/) do |f, _|
|
16
|
+
at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
|
17
|
+
end
|
18
|
+
|
19
|
+
at.add_mapping(/pt_testcase.rb/) do |f, _|
|
20
|
+
at.files_matching(/test_.*rb$/)
|
21
|
+
end
|
22
|
+
|
23
|
+
%w(TestEnvironment TestStackState).each do |klass|
|
24
|
+
at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
|
25
|
+
end
|
26
|
+
|
27
|
+
%w(TestRubyParser TestParseTree).each do |klass| # HACK
|
28
|
+
at.extra_class_map[klass] = "test/test_ruby_parser.rb"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Autotest.add_hook :run_command do |at|
|
33
|
+
system "rake parser"
|
34
|
+
end
|
35
|
+
|
36
|
+
class Autotest
|
37
|
+
def ruby
|
38
|
+
File.expand_path "~/.multiruby/install/1.9.0-0/bin/ruby"
|
39
|
+
end
|
40
|
+
end if ENV['ONENINE']
|
41
|
+
|
42
|
+
# require 'autotest/rcov'
|
43
|
+
# Autotest::RCov.command = 'rcov_info'
|
44
|
+
# Autotest::RCov.pattern = 'test/test_ruby_lexer.rb'
|
data/History.txt
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
=== 2.0.2 / 2009-01-20
|
2
|
+
|
3
|
+
* 2 minor enhancements:
|
4
|
+
|
5
|
+
* Added gauntlet_rubyparser plugin. YAY for easy massive bug-hunting.
|
6
|
+
* Promoted Sexp's file/line/comments to sexp_processor.
|
7
|
+
|
8
|
+
* 4 bug fixes:
|
9
|
+
|
10
|
+
* Fixed and improved the readme
|
11
|
+
* Fixed lexing heredoc newlines.
|
12
|
+
* Fixed line numbers on defns.
|
13
|
+
* Fixed rdoc generation bug pointed out by hugh sasse (who rocks)
|
14
|
+
|
15
|
+
=== 2.0.1 / 2008-11-04
|
16
|
+
|
17
|
+
* 2 minor enhancements:
|
18
|
+
|
19
|
+
* Updated for changes to splat node in many contexts.
|
20
|
+
* Made PT a developer dep
|
21
|
+
|
22
|
+
=== 2.0.0 / 2008-10-22
|
23
|
+
|
24
|
+
* 1 major enhancement
|
25
|
+
|
26
|
+
* Brought on the AWESOME! 4x faster! no known lexing/parsing bugs!
|
27
|
+
|
28
|
+
* 71 minor enhancements
|
29
|
+
|
30
|
+
* 1.9: Added Fixnum#ord.
|
31
|
+
* 1.9: Added missing Regexp constants and did it so it'd work on 1.9.
|
32
|
+
* Added #store_comment and #comments
|
33
|
+
* Added StringScanner #begin_of_line?
|
34
|
+
* Added a bunch of tests for regexp escape chars, #parse_string, #read_escape, ? numbers, ? whitespace.
|
35
|
+
* Added a hack for rubinius' r2l eval bug.
|
36
|
+
* Added a new token type tSTRING that bypasses tSTRING_BEG/END entirely. Only does non-interpolated strings and then falls back to the old way. MUCH cleaner tho.
|
37
|
+
* Added bin/ruby_parse
|
38
|
+
* Added compare rule to Rakefile.
|
39
|
+
* Added coverage files/dirs to clean rule.
|
40
|
+
* Added file and line numbers to all sexp nodes. Column/ranges to come.
|
41
|
+
* Added lex_state change for lvars at the end of yylex.
|
42
|
+
* Added lexed comments to defn/defs/class/module nodes.
|
43
|
+
* Added stats gathering for yylex. Reordered yylex for avg data
|
44
|
+
* Added tSYMBOL token type and parser rule to speed up symbol lexing.
|
45
|
+
* Added tally output for getch, unread, and unread_many.
|
46
|
+
* Added tests for ambigous uminus/uplus, backtick in cmdarg, square and curly brackets, numeric gvars, eos edge cases, string quoting %<> and %%%.
|
47
|
+
* All cases throughout yylex now return directly if they match, no passthroughs.
|
48
|
+
* All lexer cases now slurp entire token in one swoop.
|
49
|
+
* All zarrays are now just empty arrays.
|
50
|
+
* Changed s(:block_arg, :blah) to :"&blah" in args sexp.
|
51
|
+
* Cleaned up lexer error handling. Now just raises all over.
|
52
|
+
* Cleaned up read_escape and regx_options
|
53
|
+
* Cleaned up tokadd_string (for some definition of cleaned).
|
54
|
+
* Converted single quoted strings to new tSTRING token type.
|
55
|
+
* Coverage is currently 94.4% on lexer.
|
56
|
+
* Done what I can to clean up heredoc lexing... still sucks.
|
57
|
+
* Flattened resbodies in rescue node. Fixed .autotest file.
|
58
|
+
* Folded lex_keywords back in now that it screams.
|
59
|
+
* Found very last instanceof ILiteralNode in the code. haha!
|
60
|
+
* Got the tests subclassing PTTC and cleaned up a lot. YAY
|
61
|
+
* Handle yield(*ary) properly
|
62
|
+
* MASSIVELY cleaned out =begin/=end comment processor.
|
63
|
+
* Massive overhaul on Keyword class. All hail the mighty Hash!
|
64
|
+
* Massively cleaned up ident= edge cases and fixed a stupid bug from jruby.
|
65
|
+
* Merged @/@@ scanner together, going to try to do the same everywhere.
|
66
|
+
* Refactored fix_arg_lex_state, common across the lexer.
|
67
|
+
* Refactored new_fcall into new_call.
|
68
|
+
* Refactored some code to get better profile numbers.
|
69
|
+
* Refactored some more #fix_arg_lex_state.
|
70
|
+
* Refactored tail of yylex into its own method.
|
71
|
+
* Removed Module#kill
|
72
|
+
* Removed Token, replaced with Sexp.
|
73
|
+
* Removed all parse_number and parse_quote tests.
|
74
|
+
* Removed argspush, argscat. YAY!
|
75
|
+
* Removed as many token_buffer.split(//)'s as possible. 1 to go.
|
76
|
+
* Removed begins from compstmts
|
77
|
+
* Removed buffer arg for tokadd_string.
|
78
|
+
* Removed crufty (?) solo '@' token... wtf was that anyhow?
|
79
|
+
* Removed most jruby/stringio cruft from StringScanner.
|
80
|
+
* Removed one unread_many... 2 to go. They're harder.
|
81
|
+
* Removed store_comment, now done directly.
|
82
|
+
* Removed token_buffer. Now I just use token ivar.
|
83
|
+
* Removed use of s() from lexer. Changed the way line numbers are gathered.
|
84
|
+
* Renamed *qwords to *awords.
|
85
|
+
* Renamed StringScanner to RPStringScanner (a subclass) to fix namespace trashing.
|
86
|
+
* Renamed parse to process and aliased to parse.
|
87
|
+
* Renamed token_buffer to string_buffer since that arcane shit still needs it.
|
88
|
+
* Resolved the rest of the lexing issues I brought up w/ ruby-core.
|
89
|
+
* Revamped tokadd_escape.
|
90
|
+
* Rewrote Keyword and KWtable.
|
91
|
+
* Rewrote RubyLexer using StringScanner.
|
92
|
+
* Rewrote tokadd_escape. 79 lines down to 21.
|
93
|
+
* Split out lib/ruby_parser_extras.rb so lexer is standalone.
|
94
|
+
* Started to clean up the parser and make it as skinny as possible
|
95
|
+
* Stripped out as much code as possible.
|
96
|
+
* Stripped yylex of some dead code.
|
97
|
+
* Switched from StringIO to StringScanner.
|
98
|
+
* Updated rakefile for new hoe.
|
99
|
+
* Uses pure ruby racc if ENV['PURE_RUBY'], otherwise use c.
|
100
|
+
* Wrote a ton of lexer tests. Coverage is as close to 100% as possible.
|
101
|
+
* Wrote args to clean up the big nasty args processing grammar section.
|
102
|
+
* lex_strterm is now a plain array, removed RubyLexer#s(...).
|
103
|
+
* yield and super now flatten args.
|
104
|
+
|
105
|
+
* 21+ bug fixes:
|
106
|
+
|
107
|
+
* I'm sure this list is missing a lot:
|
108
|
+
* Fixed 2 bugs both involving attrasgn (and ilk) esp when lhs is an array.
|
109
|
+
* Fixed a bug in the lexer for strings with single digit hex escapes.
|
110
|
+
* Fixed a bug parsing: a (args) { expr }... the space caused a different route to be followed and all hell broke loose.
|
111
|
+
* Fixed a bug with x\n=beginvar not putting begin back.
|
112
|
+
* Fixed attrasgn to have arglists, not arrays.
|
113
|
+
* Fixed bug in defn/defs with block fixing.
|
114
|
+
* Fixed class/module's name slot if colon2/3.
|
115
|
+
* Fixed dstr with empty interpolation body.
|
116
|
+
* Fixed for 1.9 string/char changes.
|
117
|
+
* Fixed lexer BS wrt determining token type of words.
|
118
|
+
* Fixed lexer BS wrt pass through values and lexing words. SO STUPID.
|
119
|
+
* Fixed lexing of floats.
|
120
|
+
* Fixed lexing of identifiers followed by equals. I hope.
|
121
|
+
* Fixed masgn with splat on lhs
|
122
|
+
* Fixed new_super to deal with block_pass correctly.
|
123
|
+
* Fixed parser's treatment of :colon2 and :colon3.
|
124
|
+
* Fixed regexp scanning of escaped numbers, ANY number is valid, not just octs.
|
125
|
+
* Fixed string scanning of escaped octs, allowing 1-3 chars.
|
126
|
+
* Fixed unescape for \n
|
127
|
+
* Fixed: omg this is stupid. '()' was returning bare nil
|
128
|
+
* Fixed: remove_begin now goes to the end, not sure why it didn't before.
|
129
|
+
|
130
|
+
=== 1.0.0 / 2007-12-20
|
131
|
+
|
132
|
+
* 1 major enhancement
|
133
|
+
* Birthday!
|
134
|
+
|
data/Manifest.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
.autotest
|
2
|
+
History.txt
|
3
|
+
Manifest.txt
|
4
|
+
README.txt
|
5
|
+
Rakefile
|
6
|
+
bin/ruby_parse
|
7
|
+
lib/gauntlet_rubyparser.rb
|
8
|
+
lib/ruby_lexer.rb
|
9
|
+
lib/ruby_parser.y
|
10
|
+
lib/ruby_parser_extras.rb
|
11
|
+
test/test_ruby_lexer.rb
|
12
|
+
test/test_ruby_parser.rb
|
13
|
+
test/test_ruby_parser_extras.rb
|
data/README.txt
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
= ruby_parser
|
2
|
+
|
3
|
+
* http://parsetree.rubyforge.org/
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
|
8
|
+
racc--which does by default use a C extension). RP's output is
|
9
|
+
the same as ParseTree's output: s-expressions using ruby's arrays and
|
10
|
+
base types.
|
11
|
+
|
12
|
+
As an example:
|
13
|
+
|
14
|
+
def conditional1(arg1)
|
15
|
+
if arg1 == 0 then
|
16
|
+
return 1
|
17
|
+
end
|
18
|
+
return 0
|
19
|
+
end
|
20
|
+
|
21
|
+
becomes:
|
22
|
+
|
23
|
+
s(:defn, :conditional1,
|
24
|
+
s(:args, :arg1),
|
25
|
+
s(:scope,
|
26
|
+
s(:block,
|
27
|
+
s(:if,
|
28
|
+
s(:call, s(:lvar, :arg1), :==, s(:arglist, s(:lit, 0))),
|
29
|
+
s(:return, s(:lit, 1)),
|
30
|
+
nil),
|
31
|
+
s(:return, s(:lit, 0)))))
|
32
|
+
|
33
|
+
== FEATURES/PROBLEMS:
|
34
|
+
|
35
|
+
* Pure ruby, no compiles.
|
36
|
+
* Includes preceding comment data for defn/defs/class/module nodes!
|
37
|
+
* Incredibly simple interface.
|
38
|
+
* Output is 100% equivalent to ParseTree.
|
39
|
+
* Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
|
40
|
+
* Known Issue: Speed is now pretty good, but can always improve:
|
41
|
+
* RP parses a corpus of 3702 files in 125s (avg 108 Kb/s)
|
42
|
+
* MRI+PT parsed the same in 67.38s (avg 200.89 Kb/s)
|
43
|
+
* Known Issue: Code is much better, but still has a long way to go.
|
44
|
+
* Known Issue: Totally awesome.
|
45
|
+
* Known Issue: line number values can be slightly off. Parsing LR sucks.
|
46
|
+
|
47
|
+
== SYNOPSIS:
|
48
|
+
|
49
|
+
RubyParser.new.parse "1+1"
|
50
|
+
# => s(:call, s(:lit, 1), :+, s(:array, s(:lit, 1)))
|
51
|
+
|
52
|
+
== REQUIREMENTS:
|
53
|
+
|
54
|
+
* ruby. woot.
|
55
|
+
* sexp_processor for Sexp and SexpProcessor classes.
|
56
|
+
* ParseTree for testing.
|
57
|
+
* racc full package for parser development (compiling .y to .rb).
|
58
|
+
|
59
|
+
== INSTALL:
|
60
|
+
|
61
|
+
* sudo gem install ruby_parser
|
62
|
+
|
63
|
+
== LICENSE:
|
64
|
+
|
65
|
+
(The MIT License)
|
66
|
+
|
67
|
+
Copyright (c) 2007-2008 Ryan Davis
|
68
|
+
|
69
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
70
|
+
a copy of this software and associated documentation files (the
|
71
|
+
'Software'), to deal in the Software without restriction, including
|
72
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
73
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
74
|
+
permit persons to whom the Software is furnished to do so, subject to
|
75
|
+
the following conditions:
|
76
|
+
|
77
|
+
The above copyright notice and this permission notice shall be
|
78
|
+
included in all copies or substantial portions of the Software.
|
79
|
+
|
80
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
81
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
82
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
83
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
84
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
85
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
86
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,155 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'hoe'
|
5
|
+
|
6
|
+
Hoe.add_include_dirs("../../ParseTree/dev/lib",
|
7
|
+
"../../ParseTree/dev/test",
|
8
|
+
"../../RubyInline/dev/lib",
|
9
|
+
"../../sexp_processor/dev/lib")
|
10
|
+
|
11
|
+
require './lib/ruby_parser_extras.rb'
|
12
|
+
|
13
|
+
hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |parser|
|
14
|
+
parser.rubyforge_name = 'parsetree'
|
15
|
+
parser.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
|
16
|
+
parser.developer('Fabio Kung', 'fabio.kung@gmail.com')
|
17
|
+
|
18
|
+
parser.extra_dev_deps << 'ParseTree'
|
19
|
+
parser.extra_deps << ['sexp_processor', '>= 3.0.1']
|
20
|
+
end
|
21
|
+
|
22
|
+
hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
|
23
|
+
|
24
|
+
[:default, :multi, :test].each do |t|
|
25
|
+
task t => :parser
|
26
|
+
end
|
27
|
+
|
28
|
+
path = "pkg/ruby_parser-#{RubyParser::VERSION}"
|
29
|
+
task path => :parser do
|
30
|
+
Dir.chdir path do
|
31
|
+
sh "rake parser"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
desc "build the parser"
|
36
|
+
task :parser => ["lib/ruby_parser.rb"]
|
37
|
+
|
38
|
+
rule '.rb' => '.y' do |t|
|
39
|
+
# -v = verbose
|
40
|
+
# -t = debugging parser ~4% reduction in speed -- keep for now
|
41
|
+
# -l = no-line-convert
|
42
|
+
sh "racc -v -t -l -o #{t.name} #{t.source}"
|
43
|
+
end
|
44
|
+
|
45
|
+
task :clean do
|
46
|
+
rm_rf(Dir["**/*~"] +
|
47
|
+
Dir["**/*.diff"] +
|
48
|
+
Dir["coverage.info"] +
|
49
|
+
Dir["coverage"] +
|
50
|
+
Dir["lib/ruby_parser.rb"] +
|
51
|
+
Dir["lib/*.output"])
|
52
|
+
end
|
53
|
+
|
54
|
+
def next_num(glob)
|
55
|
+
num = Dir[glob].max[/\d+/].to_i + 1
|
56
|
+
end
|
57
|
+
|
58
|
+
begin
|
59
|
+
require 'rcov/rcovtask'
|
60
|
+
Rcov::RcovTask.new do |t|
|
61
|
+
pattern = ENV['PATTERN'] || 'test/test_ruby_*.rb'
|
62
|
+
|
63
|
+
t.test_files = FileList[pattern]
|
64
|
+
t.verbose = true
|
65
|
+
t.rcov_opts << "--threshold 80"
|
66
|
+
t.rcov_opts << "--no-color"
|
67
|
+
end
|
68
|
+
rescue LoadError
|
69
|
+
# skip
|
70
|
+
end
|
71
|
+
|
72
|
+
desc "Compares PT to RP and deletes all files that match"
|
73
|
+
task :compare do
|
74
|
+
files = Dir["unit/**/*.rb"]
|
75
|
+
puts "Parsing #{files.size} files"
|
76
|
+
files.each do |file|
|
77
|
+
puts file
|
78
|
+
system "./cmp.rb -q #{file} && rm #{file}"
|
79
|
+
end
|
80
|
+
system 'find -d unit -type d -empty -exec rmdir {} \;'
|
81
|
+
end
|
82
|
+
|
83
|
+
desc "Compares PT to RP and stops on first failure"
|
84
|
+
task :find_bug do
|
85
|
+
files = Dir["unit/**/*.rb"]
|
86
|
+
puts "Parsing #{files.size} files"
|
87
|
+
files.each do |file|
|
88
|
+
puts file
|
89
|
+
sh "./cmp.rb -q #{file}"
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
task :sort do
|
94
|
+
sh 'grepsort "^ +def" lib/ruby_lexer.rb'
|
95
|
+
sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
|
96
|
+
end
|
97
|
+
|
98
|
+
task :rcov_info => :parser do
|
99
|
+
pattern = ENV['PATTERN'] || "test/test_*.rb"
|
100
|
+
ruby "-Ilib -S rcov --text-report --save coverage.info #{pattern}"
|
101
|
+
end
|
102
|
+
|
103
|
+
task :rcov_overlay do
|
104
|
+
rcov, eol = Marshal.load(File.read("coverage.info")).last[ENV["FILE"]], 1
|
105
|
+
puts rcov[:lines].zip(rcov[:coverage]).map { |line, coverage|
|
106
|
+
bol, eol = eol, eol + line.length
|
107
|
+
[bol, eol, "#ffcccc"] unless coverage
|
108
|
+
}.compact.inspect
|
109
|
+
end
|
110
|
+
|
111
|
+
task :loc do
|
112
|
+
loc1 = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
|
113
|
+
flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
|
114
|
+
loc2 = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
|
115
|
+
flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
|
116
|
+
|
117
|
+
loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
|
118
|
+
|
119
|
+
puts "1.0.0: loc = #{loc1} flog = #{flog1}"
|
120
|
+
puts "dev : loc = #{loc2} flog = #{flog2}"
|
121
|
+
puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
|
122
|
+
end
|
123
|
+
|
124
|
+
desc "Validate against all normal files in unit dir"
|
125
|
+
task :validate do
|
126
|
+
sh "./cmp.rb unit/*.rb"
|
127
|
+
end
|
128
|
+
|
129
|
+
def run_and_log cmd, prefix
|
130
|
+
files = ENV['FILES'] || 'unit/*.rb'
|
131
|
+
p, x = prefix, "txt"
|
132
|
+
n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
|
133
|
+
f = "#{p}.#{n}.#{x}"
|
134
|
+
|
135
|
+
sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
|
136
|
+
|
137
|
+
puts File.read(f)
|
138
|
+
end
|
139
|
+
|
140
|
+
desc "Benchmark against all normal files in unit dir"
|
141
|
+
task :benchmark do
|
142
|
+
run_and_log "ruby", "benchmark"
|
143
|
+
end
|
144
|
+
|
145
|
+
desc "Profile against all normal files in unit dir"
|
146
|
+
task :profile do
|
147
|
+
run_and_log "zenprofile", "profile"
|
148
|
+
end
|
149
|
+
|
150
|
+
desc "what was that command again?"
|
151
|
+
task :huh? do
|
152
|
+
puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
|
153
|
+
end
|
154
|
+
|
155
|
+
# vim: syntax=Ruby
|
data/bin/ruby_parse
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
#!/usr/bin/ruby -s
|
2
|
+
|
3
|
+
$q ||= false
|
4
|
+
$g ||= false
|
5
|
+
|
6
|
+
require 'rubygems'
|
7
|
+
require 'ruby_parser'
|
8
|
+
require 'pp'
|
9
|
+
|
10
|
+
good = bad = 0
|
11
|
+
|
12
|
+
multi = ARGV.size != 1
|
13
|
+
total_time = 0
|
14
|
+
total_loc = 0
|
15
|
+
total_kbytes = 0
|
16
|
+
times = {}
|
17
|
+
locs = {}
|
18
|
+
kbytes = {}
|
19
|
+
|
20
|
+
begin
|
21
|
+
ARGV.each do |file|
|
22
|
+
rp = RubyParser.new
|
23
|
+
loc = `wc -l #{file}`.strip.to_i
|
24
|
+
size = `wc -c #{file}`.strip.to_i / 1024.0
|
25
|
+
locs[file] = loc
|
26
|
+
kbytes[file] = size
|
27
|
+
total_loc += loc
|
28
|
+
total_kbytes += size
|
29
|
+
if $q then
|
30
|
+
$stderr.print "."
|
31
|
+
else
|
32
|
+
warn "# file = #{file} loc = #{loc}"
|
33
|
+
end
|
34
|
+
GC.start if $g
|
35
|
+
|
36
|
+
t = Time.now
|
37
|
+
begin
|
38
|
+
begin
|
39
|
+
rp.reset
|
40
|
+
r = rp.parse(File.read(file), file)
|
41
|
+
pp r unless $q
|
42
|
+
good += 1
|
43
|
+
rescue SyntaxError => e
|
44
|
+
warn "SyntaxError for #{file}: #{e.message}"
|
45
|
+
bad += 1
|
46
|
+
end
|
47
|
+
rescue => e
|
48
|
+
warn "#{e.backtrace.first} #{e.inspect.gsub(/\n/, ' ')} for #{file}"
|
49
|
+
warn " #{e.backtrace.join("\n ")}"
|
50
|
+
bad += 1
|
51
|
+
end
|
52
|
+
|
53
|
+
t = Time.now - t
|
54
|
+
times[file] = t
|
55
|
+
total_time += t
|
56
|
+
end
|
57
|
+
rescue Interrupt
|
58
|
+
# do nothing
|
59
|
+
end
|
60
|
+
|
61
|
+
warn "done"
|
62
|
+
|
63
|
+
total = 0
|
64
|
+
times.values.each do |t|
|
65
|
+
total += t
|
66
|
+
end
|
67
|
+
|
68
|
+
puts
|
69
|
+
puts "good = #{good} bad = #{bad}" if multi
|
70
|
+
puts
|
71
|
+
|
72
|
+
format = "%5.2fs:%9.2f l/s:%8.2f Kb/s:%5d Kb:%5d loc:%s"
|
73
|
+
|
74
|
+
times.sort_by { |f, t| -t }.each do |f, t|
|
75
|
+
next if t < 0.005
|
76
|
+
loc = locs[f]
|
77
|
+
size = kbytes[f]
|
78
|
+
puts format % [t, loc / t, size / t, size, loc, f]
|
79
|
+
end
|
80
|
+
|
81
|
+
puts
|
82
|
+
|
83
|
+
puts format % [total_time,
|
84
|
+
total_loc / total_time,
|
85
|
+
total_kbytes / total_time,
|
86
|
+
total_kbytes,
|
87
|
+
total_loc,
|
88
|
+
"TOTAL"] unless total_time == 0
|
@@ -0,0 +1,120 @@
|
|
1
|
+
#!/usr/bin/ruby -ws
|
2
|
+
|
3
|
+
$f ||= false
|
4
|
+
|
5
|
+
$:.unshift "../../ruby_parser/dev/lib"
|
6
|
+
$:.unshift "../../ruby2ruby/dev/lib"
|
7
|
+
|
8
|
+
require 'rubygems'
|
9
|
+
require 'ruby2ruby'
|
10
|
+
require 'ruby_parser'
|
11
|
+
|
12
|
+
require 'gauntlet'
|
13
|
+
|
14
|
+
class RubyParserGauntlet < Gauntlet
|
15
|
+
def initialize
|
16
|
+
super
|
17
|
+
|
18
|
+
self.data = Hash.new { |h,k| h[k] = {} }
|
19
|
+
old_data = load_yaml data_file
|
20
|
+
self.data.merge! old_data
|
21
|
+
end
|
22
|
+
|
23
|
+
def should_skip? name
|
24
|
+
if $f then
|
25
|
+
if Hash === data[name] then
|
26
|
+
! data[name].empty?
|
27
|
+
else
|
28
|
+
data[name]
|
29
|
+
end
|
30
|
+
else
|
31
|
+
data[name] == true # yes, == true on purpose
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def diff_pp o1, o2
|
36
|
+
require 'pp'
|
37
|
+
|
38
|
+
File.open("/tmp/a.#{$$}", "w") do |f|
|
39
|
+
PP.pp o1, f
|
40
|
+
end
|
41
|
+
|
42
|
+
File.open("/tmp/b.#{$$}", "w") do |f|
|
43
|
+
PP.pp o2, f
|
44
|
+
end
|
45
|
+
|
46
|
+
`diff -u /tmp/a.#{$$} /tmp/b.#{$$}`
|
47
|
+
ensure
|
48
|
+
File.unlink "/tmp/a.#{$$}" rescue nil
|
49
|
+
File.unlink "/tmp/b.#{$$}" rescue nil
|
50
|
+
end
|
51
|
+
|
52
|
+
def broke name, file, msg
|
53
|
+
warn "bad"
|
54
|
+
self.data[name][file] = msg
|
55
|
+
self.dirty = true
|
56
|
+
end
|
57
|
+
|
58
|
+
def process path, name
|
59
|
+
begin
|
60
|
+
$stderr.print " #{path}: "
|
61
|
+
rp = RubyParser.new
|
62
|
+
r2r = Ruby2Ruby.new
|
63
|
+
|
64
|
+
old_ruby = File.read(path)
|
65
|
+
|
66
|
+
begin
|
67
|
+
old_sexp = rp.process old_ruby
|
68
|
+
rescue Racc::ParseError => e
|
69
|
+
self.data[name][path] = :unparsable
|
70
|
+
self.dirty = true
|
71
|
+
return
|
72
|
+
end
|
73
|
+
|
74
|
+
new_ruby = r2r.process old_sexp.deep_clone
|
75
|
+
|
76
|
+
begin
|
77
|
+
new_sexp = rp.process new_ruby
|
78
|
+
rescue Racc::ParseError => e
|
79
|
+
broke name, path, "couldn't parse new_ruby: #{e.message.strip}"
|
80
|
+
return
|
81
|
+
end
|
82
|
+
|
83
|
+
if old_sexp != new_sexp then
|
84
|
+
broke name, path, diff_pp(old_sexp, new_sexp)
|
85
|
+
return
|
86
|
+
end
|
87
|
+
|
88
|
+
self.data[name][path] = true
|
89
|
+
self.dirty = true
|
90
|
+
|
91
|
+
warn "good"
|
92
|
+
rescue Interrupt
|
93
|
+
puts "User cancelled"
|
94
|
+
exit 1
|
95
|
+
rescue Exception => e
|
96
|
+
broke name, path, " UNKNOWN ERROR: #{e}: #{e.message.strip}"
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def run name
|
101
|
+
warn name
|
102
|
+
Dir["**/*.rb"].sort.each do |path|
|
103
|
+
next if path =~ /gemspec.rb/ # HACK
|
104
|
+
next if data[name][path] == true
|
105
|
+
process path, name
|
106
|
+
end
|
107
|
+
|
108
|
+
if self.data[name].values.all? { |v| v == true } then
|
109
|
+
warn " ALL GOOD!"
|
110
|
+
self.data[name] = true
|
111
|
+
self.dirty = true
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
filter = ARGV.shift
|
117
|
+
filter = Regexp.new filter if filter
|
118
|
+
|
119
|
+
gauntlet = RubyParserGauntlet.new
|
120
|
+
gauntlet.run_the_gauntlet filter
|