ruby_parser 1.0.0 → 2.0.0

data/.autotest

@@ -1,9 +1,14 @@
 # -*- ruby -*-
 
+require 'autotest/restart'
+
 Autotest.add_hook :initialize do |at|
   at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
-  at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test"
-  at.exceptions << 'unit'
+  at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.add_exception 'unit'
+  at.add_exception 'coverage'
+  at.add_exception '.diff'
+  at.add_exception 'coverage.info'
 
   at.unit_diff = "unit_diff -u -b"
 
@@ -12,10 +17,28 @@ Autotest.add_hook :initialize do |at|
   end
 
   at.add_mapping(/pt_testcase.rb/) do |f, _|
-    at.files_matching(/^test.*rb$/)
+    at.files_matching(/test_.*rb$/)
+  end
+
+  %w(TestEnvironment TestStackState).each do |klass|
+    at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
+  end
+
+  %w(TestRubyParser TestParseTree).each do |klass| # HACK
+    at.extra_class_map[klass] = "test/test_ruby_parser.rb"
   end
 end
 
 Autotest.add_hook :run_command do |at|
   system "rake parser"
 end
+
+class Autotest
+  def ruby
+    File.expand_path "~/.multiruby/install/1.9.0-0/bin/ruby"
+  end
+end if ENV['ONENINE']
+
+# require 'autotest/rcov'
+# Autotest::RCov.command = 'rcov_info'
+# Autotest::RCov.pattern = 'test/test_ruby_lexer.rb'

data/History.txt

@@ -1,3 +1,111 @@
+== 2.0.0 / 2008-10-22
+
+* 1 major enhancement
+
+  * Brought on the AWESOME! 4x faster! no known lexing/parsing bugs!
+
+* 71 minor enhancements
+
+  * 1.9: Added Fixnum#ord.
+  * 1.9: Added missing Regexp constants and did it so it'd work on 1.9.
+  * Added #store_comment and #comments
+  * Added StringScanner #begin_of_line?
+  * Added a bunch of tests for regexp escape chars, #parse_string, #read_escape, ? numbers, ? whitespace.
+  * Added a hack for rubinius' r2l eval bug.
+  * Added a new token type tSTRING that bypasses tSTRING_BEG/END entirely. Only does non-interpolated strings and then falls back to the old way. MUCH cleaner tho.
+  * Added bin/ruby_parse
+  * Added compare rule to Rakefile.
+  * Added coverage files/dirs to clean rule.
+  * Added file and line numbers to all sexp nodes. Column/ranges to come.
+  * Added lex_state change for lvars at the end of yylex.
+  * Added lexed comments to defn/defs/class/module nodes.
+  * Added stats gathering for yylex. Reordered yylex for avg data
+  * Added tSYMBOL token type and parser rule to speed up symbol lexing.
+  * Added tally output for getch, unread, and unread_many.
+  * Added tests for ambigous uminus/uplus, backtick in cmdarg, square and curly brackets, numeric gvars, eos edge cases, string quoting %<> and %%%.
+  * All cases throughout yylex now return directly if they match, no passthroughs.
+  * All lexer cases now slurp entire token in one swoop.
+  * All zarrays are now just empty arrays.
+  * Changed s(:block_arg, :blah) to :"&blah" in args sexp.
+  * Cleaned up lexer error handling. Now just raises all over.
+  * Cleaned up read_escape and regx_options
+  * Cleaned up tokadd_string (for some definition of cleaned).
+  * Converted single quoted strings to new tSTRING token type.
+  * Coverage is currently 94.4% on lexer.
+  * Done what I can to clean up heredoc lexing... still sucks.
+  * Flattened resbodies in rescue node. Fixed .autotest file.
+  * Folded lex_keywords back in now that it screams.
+  * Found very last instanceof ILiteralNode in the code. haha!
+  * Got the tests subclassing PTTC and cleaned up a lot. YAY
+  * Handle yield(*ary) properly
+  * MASSIVELY cleaned out =begin/=end comment processor.
+  * Massive overhaul on Keyword class. All hail the mighty Hash!
+  * Massively cleaned up ident= edge cases and fixed a stupid bug from jruby.
+  * Merged @/@@ scanner together, going to try to do the same everywhere.
+  * Refactored fix_arg_lex_state, common across the lexer.
+  * Refactored new_fcall into new_call.
+  * Refactored some code to get better profile numbers.
+  * Refactored some more #fix_arg_lex_state.
+  * Refactored tail of yylex into its own method.
+  * Removed Module#kill
+  * Removed Token, replaced with Sexp.
+  * Removed all parse_number and parse_quote tests.
+  * Removed argspush, argscat. YAY!
+  * Removed as many token_buffer.split(//)'s as possible. 1 to go.
+  * Removed begins from compstmts
+  * Removed buffer arg for tokadd_string.
+  * Removed crufty (?) solo '@' token... wtf was that anyhow?
+  * Removed most jruby/stringio cruft from StringScanner.
+  * Removed one unread_many... 2 to go. They're harder.
+  * Removed store_comment, now done directly.
+  * Removed token_buffer. Now I just use token ivar.
+  * Removed use of s() from lexer. Changed the way line numbers are gathered.
+  * Renamed *qwords to *awords.
+  * Renamed StringScanner to RPStringScanner (a subclass) to fix namespace trashing.
+  * Renamed parse to process and aliased to parse.
+  * Renamed token_buffer to string_buffer since that arcane shit still needs it.
+  * Resolved the rest of the lexing issues I brought up w/ ruby-core.
+  * Revamped tokadd_escape.
+  * Rewrote Keyword and KWtable.
+  * Rewrote RubyLexer using StringScanner.
+  * Rewrote tokadd_escape. 79 lines down to 21.
+  * Split out lib/ruby_parser_extras.rb so lexer is standalone.
+  * Started to clean up the parser and make it as skinny as possible
+  * Stripped out as much code as possible.
+  * Stripped yylex of some dead code.
+  * Switched from StringIO to StringScanner.
+  * Updated rakefile for new hoe.
+  * Uses pure ruby racc if ENV['PURE_RUBY'], otherwise use c.
+  * Wrote a ton of lexer tests. Coverage is as close to 100% as possible.
+  * Wrote args to clean up the big nasty args processing grammar section.
+  * lex_strterm is now a plain array, removed RubyLexer#s(...).
+  * yield and super now flatten args.
+
+* 21+ bug fixes:
+
+  * I'm sure this list is missing a lot:
+  * Fixed 2 bugs both involving attrasgn (and ilk) esp when lhs is an array.
+  * Fixed a bug in the lexer for strings with single digit hex escapes.
+  * Fixed a bug parsing: a (args) { expr }... the space caused a different route to be followed and all hell broke loose.
+  * Fixed a bug with x\n=beginvar not putting begin back.
+  * Fixed attrasgn to have arglists, not arrays.
+  * Fixed bug in defn/defs with block fixing.
+  * Fixed class/module's name slot if colon2/3.
+  * Fixed dstr with empty interpolation body.
+  * Fixed for 1.9 string/char changes.
+  * Fixed lexer BS wrt determining token type of words.
+  * Fixed lexer BS wrt pass through values and lexing words. SO STUPID.
+  * Fixed lexing of floats.
+  * Fixed lexing of identifiers followed by equals. I hope.
+  * Fixed masgn with splat on lhs
+  * Fixed new_super to deal with block_pass correctly.
+  * Fixed parser's treatment of :colon2 and :colon3.
+  * Fixed regexp scanning of escaped numbers, ANY number is valid, not just octs.
+  * Fixed string scanning of escaped octs, allowing 1-3 chars.
+  * Fixed unescape for \n
+  * Fixed: omg this is stupid. '()' was returning bare nil
+  * Fixed: remove_begin now goes to the end, not sure why it didn't before.
+
 == 1.0.0 / 2007-12-20
 
 * 1 major enhancement

data/Manifest.txt

@@ -3,7 +3,10 @@ History.txt
 Manifest.txt
 README.txt
 Rakefile
+bin/ruby_parse
 lib/ruby_lexer.rb
 lib/ruby_parser.y
+lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb
+test/test_ruby_parser_extras.rb

data/README.txt

@@ -12,6 +12,7 @@ base types.
 == FEATURES/PROBLEMS:
 
 * Pure ruby, no compiles.
+* Includes preceding comment data for defn/defs/class/module nodes!
 * Incredibly simple interface.
 * Output is 100% equivalent to ParseTree.
   * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
@@ -20,7 +21,6 @@ base types.
 * Known Issue: I don't currently support newline nodes.
 * Known Issue: Totally awesome.
 * Known Issue: dasgn_curr decls can be out of order from ParseTree's.
-* TODO: Add comment nodes.
 
 == SYNOPSIS:
 

data/Rakefile

@@ -2,31 +2,27 @@
 
 require 'rubygems'
 require 'hoe'
-require './lib/ruby_lexer.rb'
 
-hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |p|
-  p.rubyforge_name = 'parsetree'
-  p.author = 'Ryan Davis'
-  p.email = 'ryand-ruby@zenspider.com'
-  p.summary = p.paragraphs_of('README.txt', 2).join("\n\n")
-  p.description = p.paragraphs_of('README.txt', 2..6).join("\n\n")
-  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[-1]
-  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
-  p.extra_deps << 'ParseTree'
-end
+Hoe.add_include_dirs("../../ParseTree/dev/lib",
+                     "../../ParseTree/dev/test",
+                     "../../RubyInline/dev/lib",
+                     "../../sexp_processor/dev/lib")
 
-hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
+require './lib/ruby_parser_extras.rb'
 
-module Rake::TaskManager
-  def all_tasks
-    @tasks
-  end
+hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |parser|
+  parser.rubyforge_name = 'parsetree'
+  parser.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
+  parser.extra_deps << 'ParseTree'
+
+  parser.extra_deps << ['sexp_processor', '>= 3.0.0']
 end
 
-Rake.application.all_tasks["default"].prerequisites.clear
+hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
 
-task :default => :parser
-task :test => :parser
+[:default, :multi, :test].each do |t|
+  task t => :parser
+end
 
 path = "pkg/ruby_parser-#{RubyParser::VERSION}"
 task path => :parser do
@@ -35,22 +31,124 @@ task path => :parser do
   end
 end
 
+desc "build the parser"
 task :parser => ["lib/ruby_parser.rb"]
 
 rule '.rb' => '.y' do |t|
-  sh "racc -g -o #{t.name} #{t.source}"
+  # -v = verbose
+  # -t = debugging parser ~4% reduction in speed -- keep for now
+  # -l = no-line-convert
+  sh "racc -v -t -l -o #{t.name} #{t.source}"
 end
 
 task :clean do
-  rm_f(Dir["**/*~"] +
-       Dir["**/*.diff"] +
-       Dir["lib/ruby_parser.rb"] +
-       Dir["lib/*.output"])
+  rm_rf(Dir["**/*~"] +
+        Dir["**/*.diff"] +
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/ruby_parser.rb"] +
+        Dir["lib/*.output"])
+end
+
+def next_num(glob)
+  num = Dir[glob].max[/\d+/].to_i + 1
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    pattern = ENV['PATTERN'] || 'test/test_ruby_*.rb'
+
+    t.test_files = FileList[pattern]
+    t.verbose = true
+    t.rcov_opts << "--threshold 80"
+    t.rcov_opts << "--no-color"
+  end
+rescue LoadError
+  # skip
+end
+
+desc "Compares PT to RP and deletes all files that match"
+task :compare do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    system "./cmp.rb -q #{file} && rm #{file}"
+  end
+  system 'find -d unit -type d -empty -exec rmdir {} \;'
+end
+
+desc "Compares PT to RP and stops on first failure"
+task :find_bug do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    sh "./cmp.rb -q #{file}"
+  end
+end
+
+task :sort do
+  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
+  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
 end
 
-# require 'rcov/rcovtask'
-# Rcov::RcovTask.new do |t|
-#   t.test_files = FileList['test/test_ruby_lexer.rb']
-# end
+task :rcov_info => :parser do
+  pattern = ENV['PATTERN'] || "test/test_*.rb"
+  ruby "-Ilib -S rcov --text-report --save coverage.info #{pattern}"
+end
+
+task :rcov_overlay do
+  rcov, eol = Marshal.load(File.read("coverage.info")).last[ENV["FILE"]], 1
+  puts rcov[:lines].zip(rcov[:coverage]).map { |line, coverage|
+    bol, eol = eol, eol + line.length
+    [bol, eol, "#ffcccc"] unless coverage
+  }.compact.inspect
+end
+
+task :loc do
+  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
+  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
+  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
+  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+
+  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+
+  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
+  puts "dev  : loc = #{loc2} flog = #{flog2}"
+  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+end
+
+desc "Validate against all normal files in unit dir"
+task :validate do
+  sh "./cmp.rb unit/*.rb"
+end
+
+def run_and_log cmd, prefix
+  files = ENV['FILES'] || 'unit/*.rb'
+  p, x = prefix, "txt"
+  n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
+  f = "#{p}.#{n}.#{x}"
+
+  sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
+
+  puts File.read(f)
+end
+
+desc "Benchmark against all normal files in unit dir"
+task :benchmark do
+  run_and_log "ruby", "benchmark"
+end
+
+desc "Profile against all normal files in unit dir"
+task :profile do
+  run_and_log "zenprofile", "profile"
+end
+
+desc "what was that command again?"
+task :huh? do
+  puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
+end
 
 # vim: syntax=Ruby

data/bin/ruby_parse

@@ -0,0 +1,89 @@
+#!/usr/bin/ruby -s
+
+$q ||= false
+$g ||= false
+
+require 'rubygems'
+require 'ruby_parser'
+require 'pp'
+
+good = bad = 0
+
+multi = ARGV.size != 1
+total_time = 0
+total_loc = 0
+total_kbytes = 0
+times = {}
+locs = {}
+kbytes = {}
+
+begin
+  ARGV.each do |file|
+    rp = RubyParser.new
+    loc = `wc -l #{file}`.strip.to_i
+    size = `wc -c #{file}`.strip.to_i / 1024.0
+    locs[file] = loc
+    kbytes[file] = size
+    total_loc += loc
+    total_kbytes += size
+    if $q then
+      $stderr.print "."
+    else
+      warn "# file = #{file} loc = #{loc}"
+    end
+    GC.start if $g
+
+    t = Time.now
+    begin
+      begin
+        rp.reset
+        r = rp.parse(File.read(file), file)
+        pp r unless $q
+        good += 1
+      rescue SyntaxError => e
+        warn "SyntaxError for #{file}: #{e.message}"
+        bad += 1
+      end
+    rescue => e
+      warn "#{e.backtrace.first} #{e.inspect.gsub(/\n/, ' ')} for #{file}"
+      warn "  #{e.backtrace.join("\n  ")}"
+      bad += 1
+    end
+
+    t = Time.now - t
+    times[file] = t
+    total_time += t
+  end
+rescue Interrupt
+  # do nothing
+end
+
+warn "done"
+
+total = 0
+times.values.each do |t|
+  total += t
+end
+
+puts
+puts "good = #{good} bad = #{bad}" if multi
+puts
+
+format = "%5.2fs:%9.2f l/s:%8.2f Kb/s:%5d Kb:%5d loc:%s"
+
+times.sort_by { |f, t| -t }.each do |f, t|
+  next if t < 0.005
+  loc = locs[f]
+  size = kbytes[f]
+  puts format % [t, loc / t, size / t, size, loc, f]
+end
+
+puts
+
+puts format % [total_time,
+               total_loc / total_time,
+               total_kbytes / total_time,
+               total_kbytes,
+               total_loc,
+               "TOTAL"]
+