fabiokung-ruby_parser 2.0.2

data/.autotest

@@ -0,0 +1,44 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+Autotest.add_hook :initialize do |at|
+  at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
+  at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.add_exception 'unit'
+  at.add_exception 'coverage'
+  at.add_exception '.diff'
+  at.add_exception 'coverage.info'
+
+  at.unit_diff = "unit_diff -u -b"
+
+  at.add_mapping(/^lib\/.*\.y$/) do |f, _|
+    at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
+  end
+
+  at.add_mapping(/pt_testcase.rb/) do |f, _|
+    at.files_matching(/test_.*rb$/)
+  end
+
+  %w(TestEnvironment TestStackState).each do |klass|
+    at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
+  end
+
+  %w(TestRubyParser TestParseTree).each do |klass| # HACK
+    at.extra_class_map[klass] = "test/test_ruby_parser.rb"
+  end
+end
+
+Autotest.add_hook :run_command do |at|
+  system "rake parser"
+end
+
+class Autotest
+  def ruby
+    File.expand_path "~/.multiruby/install/1.9.0-0/bin/ruby"
+  end
+end if ENV['ONENINE']
+
+# require 'autotest/rcov'
+# Autotest::RCov.command = 'rcov_info'
+# Autotest::RCov.pattern = 'test/test_ruby_lexer.rb'

data/History.txt

@@ -0,0 +1,134 @@
+=== 2.0.2 / 2009-01-20
+
+* 2 minor enhancements:
+
+  * Added gauntlet_rubyparser plugin. YAY for easy massive bug-hunting.
+  * Promoted Sexp's file/line/comments to sexp_processor.
+
+* 4 bug fixes:
+
+  * Fixed and improved the readme
+  * Fixed lexing heredoc newlines.
+  * Fixed line numbers on defns.
+  * Fixed rdoc generation bug pointed out by hugh sasse (who rocks)
+
+=== 2.0.1 / 2008-11-04
+
+* 2 minor enhancements:
+
+  * Updated for changes to splat node in many contexts.
+  * Made PT a developer dep
+
+=== 2.0.0 / 2008-10-22
+
+* 1 major enhancement
+
+  * Brought on the AWESOME! 4x faster! no known lexing/parsing bugs!
+
+* 71 minor enhancements
+
+  * 1.9: Added Fixnum#ord.
+  * 1.9: Added missing Regexp constants and did it so it'd work on 1.9.
+  * Added #store_comment and #comments
+  * Added StringScanner #begin_of_line?
+  * Added a bunch of tests for regexp escape chars, #parse_string, #read_escape, ? numbers, ? whitespace.
+  * Added a hack for rubinius' r2l eval bug.
+  * Added a new token type tSTRING that bypasses tSTRING_BEG/END entirely. Only does non-interpolated strings and then falls back to the old way. MUCH cleaner tho.
+  * Added bin/ruby_parse
+  * Added compare rule to Rakefile.
+  * Added coverage files/dirs to clean rule.
+  * Added file and line numbers to all sexp nodes. Column/ranges to come.
+  * Added lex_state change for lvars at the end of yylex.
+  * Added lexed comments to defn/defs/class/module nodes.
+  * Added stats gathering for yylex. Reordered yylex for avg data
+  * Added tSYMBOL token type and parser rule to speed up symbol lexing.
+  * Added tally output for getch, unread, and unread_many.
+  * Added tests for ambigous uminus/uplus, backtick in cmdarg, square and curly brackets, numeric gvars, eos edge cases, string quoting %<> and %%%.
+  * All cases throughout yylex now return directly if they match, no passthroughs.
+  * All lexer cases now slurp entire token in one swoop.
+  * All zarrays are now just empty arrays.
+  * Changed s(:block_arg, :blah) to :"&blah" in args sexp.
+  * Cleaned up lexer error handling. Now just raises all over.
+  * Cleaned up read_escape and regx_options
+  * Cleaned up tokadd_string (for some definition of cleaned).
+  * Converted single quoted strings to new tSTRING token type.
+  * Coverage is currently 94.4% on lexer.
+  * Done what I can to clean up heredoc lexing... still sucks.
+  * Flattened resbodies in rescue node. Fixed .autotest file.
+  * Folded lex_keywords back in now that it screams.
+  * Found very last instanceof ILiteralNode in the code. haha!
+  * Got the tests subclassing PTTC and cleaned up a lot. YAY
+  * Handle yield(*ary) properly
+  * MASSIVELY cleaned out =begin/=end comment processor.
+  * Massive overhaul on Keyword class. All hail the mighty Hash!
+  * Massively cleaned up ident= edge cases and fixed a stupid bug from jruby.
+  * Merged @/@@ scanner together, going to try to do the same everywhere.
+  * Refactored fix_arg_lex_state, common across the lexer.
+  * Refactored new_fcall into new_call.
+  * Refactored some code to get better profile numbers.
+  * Refactored some more #fix_arg_lex_state.
+  * Refactored tail of yylex into its own method.
+  * Removed Module#kill
+  * Removed Token, replaced with Sexp.
+  * Removed all parse_number and parse_quote tests.
+  * Removed argspush, argscat. YAY!
+  * Removed as many token_buffer.split(//)'s as possible. 1 to go.
+  * Removed begins from compstmts
+  * Removed buffer arg for tokadd_string.
+  * Removed crufty (?) solo '@' token... wtf was that anyhow?
+  * Removed most jruby/stringio cruft from StringScanner.
+  * Removed one unread_many... 2 to go. They're harder.
+  * Removed store_comment, now done directly.
+  * Removed token_buffer. Now I just use token ivar.
+  * Removed use of s() from lexer. Changed the way line numbers are gathered.
+  * Renamed *qwords to *awords.
+  * Renamed StringScanner to RPStringScanner (a subclass) to fix namespace trashing.
+  * Renamed parse to process and aliased to parse.
+  * Renamed token_buffer to string_buffer since that arcane shit still needs it.
+  * Resolved the rest of the lexing issues I brought up w/ ruby-core.
+  * Revamped tokadd_escape.
+  * Rewrote Keyword and KWtable.
+  * Rewrote RubyLexer using StringScanner.
+  * Rewrote tokadd_escape. 79 lines down to 21.
+  * Split out lib/ruby_parser_extras.rb so lexer is standalone.
+  * Started to clean up the parser and make it as skinny as possible
+  * Stripped out as much code as possible.
+  * Stripped yylex of some dead code.
+  * Switched from StringIO to StringScanner.
+  * Updated rakefile for new hoe.
+  * Uses pure ruby racc if ENV['PURE_RUBY'], otherwise use c.
+  * Wrote a ton of lexer tests. Coverage is as close to 100% as possible.
+  * Wrote args to clean up the big nasty args processing grammar section.
+  * lex_strterm is now a plain array, removed RubyLexer#s(...).
+  * yield and super now flatten args.
+
+* 21+ bug fixes:
+
+  * I'm sure this list is missing a lot:
+  * Fixed 2 bugs both involving attrasgn (and ilk) esp when lhs is an array.
+  * Fixed a bug in the lexer for strings with single digit hex escapes.
+  * Fixed a bug parsing: a (args) { expr }... the space caused a different route to be followed and all hell broke loose.
+  * Fixed a bug with x\n=beginvar not putting begin back.
+  * Fixed attrasgn to have arglists, not arrays.
+  * Fixed bug in defn/defs with block fixing.
+  * Fixed class/module's name slot if colon2/3.
+  * Fixed dstr with empty interpolation body.
+  * Fixed for 1.9 string/char changes.
+  * Fixed lexer BS wrt determining token type of words.
+  * Fixed lexer BS wrt pass through values and lexing words. SO STUPID.
+  * Fixed lexing of floats.
+  * Fixed lexing of identifiers followed by equals. I hope.
+  * Fixed masgn with splat on lhs
+  * Fixed new_super to deal with block_pass correctly.
+  * Fixed parser's treatment of :colon2 and :colon3.
+  * Fixed regexp scanning of escaped numbers, ANY number is valid, not just octs.
+  * Fixed string scanning of escaped octs, allowing 1-3 chars.
+  * Fixed unescape for \n
+  * Fixed: omg this is stupid. '()' was returning bare nil
+  * Fixed: remove_begin now goes to the end, not sure why it didn't before.
+
+=== 1.0.0 / 2007-12-20
+
+* 1 major enhancement
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,13 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/ruby_parse
+lib/gauntlet_rubyparser.rb
+lib/ruby_lexer.rb
+lib/ruby_parser.y
+lib/ruby_parser_extras.rb
+test/test_ruby_lexer.rb
+test/test_ruby_parser.rb
+test/test_ruby_parser_extras.rb

data/README.txt

@@ -0,0 +1,86 @@
+= ruby_parser
+
+* http://parsetree.rubyforge.org/
+
+== DESCRIPTION:
+
+ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
+racc--which does by default use a C extension). RP's output is
+the same as ParseTree's output: s-expressions using ruby's arrays and
+base types.
+
+As an example:
+
+  def conditional1(arg1)
+    if arg1 == 0 then
+      return 1
+    end
+    return 0
+  end
+
+becomes:
+
+  s(:defn, :conditional1,
+   s(:args, :arg1),
+   s(:scope,
+    s(:block,
+     s(:if,
+      s(:call, s(:lvar, :arg1), :==, s(:arglist, s(:lit, 0))),
+      s(:return, s(:lit, 1)),
+      nil),
+     s(:return, s(:lit, 0)))))
+
+== FEATURES/PROBLEMS:
+
+* Pure ruby, no compiles.
+* Includes preceding comment data for defn/defs/class/module nodes!
+* Incredibly simple interface.
+* Output is 100% equivalent to ParseTree.
+  * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
+* Known Issue: Speed is now pretty good, but can always improve:
+  * RP parses a corpus of 3702 files in 125s (avg 108 Kb/s)
+  * MRI+PT parsed the same in 67.38s (avg 200.89 Kb/s)
+* Known Issue: Code is much better, but still has a long way to go.
+* Known Issue: Totally awesome.
+* Known Issue: line number values can be slightly off. Parsing LR sucks.
+
+== SYNOPSIS:
+
+  RubyParser.new.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:array, s(:lit, 1)))
+
+== REQUIREMENTS:
+
+* ruby. woot.
+* sexp_processor for Sexp and SexpProcessor classes.
+* ParseTree for testing.
+* racc full package for parser development (compiling .y to .rb).
+
+== INSTALL:
+
+* sudo gem install ruby_parser
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007-2008 Ryan Davis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,155 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.add_include_dirs("../../ParseTree/dev/lib",
+                     "../../ParseTree/dev/test",
+                     "../../RubyInline/dev/lib",
+                     "../../sexp_processor/dev/lib")
+
+require './lib/ruby_parser_extras.rb'
+
+hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |parser|
+  parser.rubyforge_name = 'parsetree'
+  parser.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
+  parser.developer('Fabio Kung', 'fabio.kung@gmail.com')
+
+  parser.extra_dev_deps << 'ParseTree'
+  parser.extra_deps << ['sexp_processor', '>= 3.0.1']
+end
+
+hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
+
+[:default, :multi, :test].each do |t|
+  task t => :parser
+end
+
+path = "pkg/ruby_parser-#{RubyParser::VERSION}"
+task path => :parser do
+  Dir.chdir path do
+    sh "rake parser"
+  end
+end
+
+desc "build the parser"
+task :parser => ["lib/ruby_parser.rb"]
+
+rule '.rb' => '.y' do |t|
+  # -v = verbose
+  # -t = debugging parser ~4% reduction in speed -- keep for now
+  # -l = no-line-convert
+  sh "racc -v -t -l -o #{t.name} #{t.source}"
+end
+
+task :clean do
+  rm_rf(Dir["**/*~"] +
+        Dir["**/*.diff"] +
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/ruby_parser.rb"] +
+        Dir["lib/*.output"])
+end
+
+def next_num(glob)
+  num = Dir[glob].max[/\d+/].to_i + 1
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |t|
+    pattern = ENV['PATTERN'] || 'test/test_ruby_*.rb'
+
+    t.test_files = FileList[pattern]
+    t.verbose = true
+    t.rcov_opts << "--threshold 80"
+    t.rcov_opts << "--no-color"
+  end
+rescue LoadError
+  # skip
+end
+
+desc "Compares PT to RP and deletes all files that match"
+task :compare do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    system "./cmp.rb -q #{file} && rm #{file}"
+  end
+  system 'find -d unit -type d -empty -exec rmdir {} \;'
+end
+
+desc "Compares PT to RP and stops on first failure"
+task :find_bug do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    sh "./cmp.rb -q #{file}"
+  end
+end
+
+task :sort do
+  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
+  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
+end
+
+task :rcov_info => :parser do
+  pattern = ENV['PATTERN'] || "test/test_*.rb"
+  ruby "-Ilib -S rcov --text-report --save coverage.info #{pattern}"
+end
+
+task :rcov_overlay do
+  rcov, eol = Marshal.load(File.read("coverage.info")).last[ENV["FILE"]], 1
+  puts rcov[:lines].zip(rcov[:coverage]).map { |line, coverage|
+    bol, eol = eol, eol + line.length
+    [bol, eol, "#ffcccc"] unless coverage
+  }.compact.inspect
+end
+
+task :loc do
+  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
+  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
+  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
+  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+
+  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+
+  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
+  puts "dev  : loc = #{loc2} flog = #{flog2}"
+  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+end
+
+desc "Validate against all normal files in unit dir"
+task :validate do
+  sh "./cmp.rb unit/*.rb"
+end
+
+def run_and_log cmd, prefix
+  files = ENV['FILES'] || 'unit/*.rb'
+  p, x = prefix, "txt"
+  n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
+  f = "#{p}.#{n}.#{x}"
+
+  sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
+
+  puts File.read(f)
+end
+
+desc "Benchmark against all normal files in unit dir"
+task :benchmark do
+  run_and_log "ruby", "benchmark"
+end
+
+desc "Profile against all normal files in unit dir"
+task :profile do
+  run_and_log "zenprofile", "profile"
+end
+
+desc "what was that command again?"
+task :huh? do
+  puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
+end
+
+# vim: syntax=Ruby

data/bin/ruby_parse

@@ -0,0 +1,88 @@
+#!/usr/bin/ruby -s
+
+$q ||= false
+$g ||= false
+
+require 'rubygems'
+require 'ruby_parser'
+require 'pp'
+
+good = bad = 0
+
+multi = ARGV.size != 1
+total_time = 0
+total_loc = 0
+total_kbytes = 0
+times = {}
+locs = {}
+kbytes = {}
+
+begin
+  ARGV.each do |file|
+    rp = RubyParser.new
+    loc = `wc -l #{file}`.strip.to_i
+    size = `wc -c #{file}`.strip.to_i / 1024.0
+    locs[file] = loc
+    kbytes[file] = size
+    total_loc += loc
+    total_kbytes += size
+    if $q then
+      $stderr.print "."
+    else
+      warn "# file = #{file} loc = #{loc}"
+    end
+    GC.start if $g
+
+    t = Time.now
+    begin
+      begin
+        rp.reset
+        r = rp.parse(File.read(file), file)
+        pp r unless $q
+        good += 1
+      rescue SyntaxError => e
+        warn "SyntaxError for #{file}: #{e.message}"
+        bad += 1
+      end
+    rescue => e
+      warn "#{e.backtrace.first} #{e.inspect.gsub(/\n/, ' ')} for #{file}"
+      warn "  #{e.backtrace.join("\n  ")}"
+      bad += 1
+    end
+
+    t = Time.now - t
+    times[file] = t
+    total_time += t
+  end
+rescue Interrupt
+  # do nothing
+end
+
+warn "done"
+
+total = 0
+times.values.each do |t|
+  total += t
+end
+
+puts
+puts "good = #{good} bad = #{bad}" if multi
+puts
+
+format = "%5.2fs:%9.2f l/s:%8.2f Kb/s:%5d Kb:%5d loc:%s"
+
+times.sort_by { |f, t| -t }.each do |f, t|
+  next if t < 0.005
+  loc = locs[f]
+  size = kbytes[f]
+  puts format % [t, loc / t, size / t, size, loc, f]
+end
+
+puts
+
+puts format % [total_time,
+               total_loc / total_time,
+               total_kbytes / total_time,
+               total_kbytes,
+               total_loc,
+               "TOTAL"] unless total_time == 0

data/lib/gauntlet_rubyparser.rb

@@ -0,0 +1,120 @@
+#!/usr/bin/ruby -ws
+
+$f ||= false
+
+$:.unshift "../../ruby_parser/dev/lib"
+$:.unshift "../../ruby2ruby/dev/lib"
+
+require 'rubygems'
+require 'ruby2ruby'
+require 'ruby_parser'
+
+require 'gauntlet'
+
+class RubyParserGauntlet < Gauntlet
+  def initialize
+    super
+
+    self.data = Hash.new { |h,k| h[k] = {} }
+    old_data = load_yaml data_file
+    self.data.merge! old_data
+  end
+
+  def should_skip? name
+    if $f then
+      if Hash === data[name] then
+        ! data[name].empty?
+      else
+        data[name]
+      end
+    else
+      data[name] == true # yes, == true on purpose
+    end
+  end
+
+  def diff_pp o1, o2
+    require 'pp'
+
+    File.open("/tmp/a.#{$$}", "w") do |f|
+      PP.pp o1, f
+    end
+
+    File.open("/tmp/b.#{$$}", "w") do |f|
+      PP.pp o2, f
+    end
+
+    `diff -u /tmp/a.#{$$} /tmp/b.#{$$}`
+  ensure
+    File.unlink "/tmp/a.#{$$}" rescue nil
+    File.unlink "/tmp/b.#{$$}" rescue nil
+  end
+
+  def broke name, file, msg
+    warn "bad"
+    self.data[name][file] = msg
+    self.dirty = true
+  end
+
+  def process path, name
+    begin
+      $stderr.print "  #{path}: "
+      rp = RubyParser.new
+      r2r = Ruby2Ruby.new
+
+      old_ruby = File.read(path)
+
+      begin
+        old_sexp = rp.process old_ruby
+      rescue Racc::ParseError => e
+        self.data[name][path] = :unparsable
+        self.dirty = true
+        return
+      end
+
+      new_ruby = r2r.process old_sexp.deep_clone
+
+      begin
+        new_sexp = rp.process new_ruby
+      rescue Racc::ParseError => e
+        broke name, path, "couldn't parse new_ruby: #{e.message.strip}"
+        return
+      end
+
+      if old_sexp != new_sexp then
+        broke name, path, diff_pp(old_sexp, new_sexp)
+        return
+      end
+
+      self.data[name][path] = true
+      self.dirty = true
+
+      warn "good"
+    rescue Interrupt
+      puts "User cancelled"
+      exit 1
+    rescue Exception => e
+      broke name, path, "    UNKNOWN ERROR: #{e}: #{e.message.strip}"
+    end
+  end
+
+  def run name
+    warn name
+    Dir["**/*.rb"].sort.each do |path|
+      next if path =~ /gemspec.rb/ # HACK
+      next if data[name][path] == true
+      process path, name
+    end
+
+    if self.data[name].values.all? { |v| v == true } then
+      warn "  ALL GOOD!"
+      self.data[name] = true
+      self.dirty = true
+    end
+  end
+end
+
+filter = ARGV.shift
+filter = Regexp.new filter if filter
+
+gauntlet = RubyParserGauntlet.new
+gauntlet.run_the_gauntlet filter