parser 0.9.alpha

data/Manifest.txt

@@ -0,0 +1,18 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/ruby_parse
+bin/ruby_parse_extract_error
+lib/gauntlet_rubyparser.rb
+lib/ruby18_parser.rb
+lib/ruby18_parser.y
+lib/ruby19_parser.rb
+lib/ruby19_parser.y
+lib/ruby_lexer.rb
+lib/ruby_parser.rb
+lib/ruby_parser_extras.rb
+test/test_ruby_lexer.rb
+test/test_ruby_parser.rb
+test/test_ruby_parser_extras.rb

data/README.txt

@@ -0,0 +1,87 @@
+= ruby_parser
+
+home :: https://github.com/seattlerb/ruby_parser
+bugs :: https://github.com/seattlerb/ruby_parser/issues
+rdoc :: http://docs.seattlerb.org/ruby_parser
+
+== DESCRIPTION:
+
+ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
+racc--which does by default use a C extension). RP's output is
+the same as ParseTree's output: s-expressions using ruby's arrays and
+base types.
+
+As an example:
+
+  def conditional1 arg1
+    return 1 if arg1 == 0
+    return 0
+  end
+
+becomes:
+
+  s(:defn, :conditional1, s(:args, :arg1),
+    s(:if,
+      s(:call, s(:lvar, :arg1), :==, s(:lit, 0)),
+      s(:return, s(:lit, 1)),
+      nil),
+    s(:return, s(:lit, 0)))
+
+== FEATURES/PROBLEMS:
+
+* Pure ruby, no compiles.
+* Includes preceding comment data for defn/defs/class/module nodes!
+* Incredibly simple interface.
+* Output is 100% equivalent to ParseTree.
+  * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
+* Known Issue: Speed is now pretty good, but can always improve:
+  * RP parses a corpus of 3702 files in 125s (avg 108 Kb/s)
+  * MRI+PT parsed the same in 67.38s (avg 200.89 Kb/s)
+* Known Issue: Code is much better, but still has a long way to go.
+* Known Issue: Totally awesome.
+* Known Issue: line number values can be slightly off. Parsing LR sucks.
+
+== SYNOPSIS:
+
+  RubyParser.new.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:lit, 1))
+
+You can also use Ruby19Parser, Ruby18Parser, or RubyParser.for_current_ruby:
+
+  RubyParser.for_current_ruby.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:lit, 1))
+
+== REQUIREMENTS:
+
+* ruby. woot.
+* sexp_processor for Sexp and SexpProcessor classes, and testing.
+* racc full package for parser development (compiling .y to .rb).
+
+== INSTALL:
+
+* sudo gem install ruby_parser
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) Ryan Davis, seattle.rb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,192 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.plugin :seattlerb
+Hoe.plugin :racc
+Hoe.plugin :isolate
+
+Hoe.add_include_dirs "../../sexp_processor/dev/lib"
+
+Hoe.spec 'parser' do
+  developer 'Peter Zotov', 'whitequark@whitequark.org'
+
+  dependency 'sexp_processor', '~> 4.1'
+
+  self.racc_flags << " -t" if plugin?(:racc) && ENV["DEBUG"]
+end
+
+file "lib/ruby18_parser.rb" => "lib/ruby18_parser.y"
+file "lib/ruby19_parser.rb" => "lib/ruby19_parser.y"
+
+file "lib/ruby_lexer.rb" => "lib/ruby_lexer.rl" do |t|
+  sh "ragel -R #{t.prerequisites.first} -o #{t.name}"
+end
+
+task :clean do
+  rm_rf(Dir["**/*~"] +
+        Dir["**/*.diff"] +
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/*.output"])
+end
+
+def next_num(glob)
+  num = Dir[glob].max[/\d+/].to_i + 1
+end
+
+desc "Compares PT to RP and deletes all files that match"
+task :compare do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    system "./cmp.rb -q #{file} && rm #{file}"
+  end
+  system 'find -d unit -type d -empty -exec rmdir {} \;'
+end
+
+desc "Compares PT to RP and stops on first failure"
+task :find_bug do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    sh "./cmp.rb -q #{file}"
+  end
+end
+
+task :sort do
+  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
+  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
+end
+
+task :loc do
+  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
+  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
+  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
+  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+
+  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+
+  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
+  puts "dev  : loc = #{loc2} flog = #{flog2}"
+  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+end
+
+desc "Validate against all normal files in unit dir"
+task :validate do
+  sh "./cmp.rb unit/*.rb"
+end
+
+def run_and_log cmd, prefix
+  files = ENV['FILES'] || 'unit/*.rb'
+  p, x = prefix, "txt"
+  n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
+  f = "#{p}.#{n}.#{x}"
+
+  sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
+
+  puts File.read(f)
+end
+
+desc "Benchmark against all normal files in unit dir"
+task :benchmark do
+  run_and_log "ruby", "benchmark"
+end
+
+desc "Profile against all normal files in unit dir"
+task :profile do
+  run_and_log "zenprofile", "profile"
+end
+
+desc "what was that command again?"
+task :huh? do
+  puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
+end
+
+task :irb => [:isolate] do
+  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
+end
+
+def (task(:phony)).timestamp
+  Time.at 0
+end
+
+task :isolate => :phony
+
+file "lib/ruby18_parser.rb" => :isolate
+file "lib/ruby19_parser.rb" => :isolate
+
+task :compare18 do
+  sh "./yack.rb lib/ruby18_parser.output > racc18.txt"
+  sh "./yack.rb parse18.output > yacc18.txt"
+  sh "diff -du racc18.txt yacc18.txt || true"
+  puts
+  sh "diff -du racc18.txt yacc18.txt | wc -l"
+end
+
+task :compare19 do
+  sh "./yack.rb lib/ruby19_parser.output > racc19.txt"
+  sh "./yack.rb parse19.output > yacc19.txt"
+  sh "diff -du racc19.txt yacc19.txt || true"
+  puts
+  sh "diff -du racc19.txt yacc19.txt | wc -l"
+end
+
+task :debug => :isolate do
+  ENV["V"] ||= "19"
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  $: << "lib"
+  require 'ruby_parser'
+  require 'pp'
+
+  parser = if ENV["V"] == "18" then
+             Ruby18Parser.new
+           else
+             Ruby19Parser.new
+           end
+
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
+  file = ENV["F"] || ENV["FILE"]
+
+  ruby = if file then
+           File.read(file)
+         else
+           file = "env"
+           ENV["R"] || ENV["RUBY"]
+         end
+
+  begin
+    pp parser.process(ruby, file, time)
+  rescue Racc::ParseError => e
+    p e
+    ss = parser.lexer.src
+    src = ss.string
+    lines = src[0..ss.pos].split(/\n/)
+    abort "on #{file}:#{lines.size}"
+  end
+end
+
+task :debug_ruby do
+  file = ENV["F"] || ENV["FILE"]
+  sh "ruby19 -cwy #{file} 2>&1 | ./yuck.rb"
+end
+
+task :extract => :isolate do
+  ENV["V"] ||= "19"
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  file = ENV["F"] || ENV["FILE"]
+
+  ruby "-Ilib", "bin/ruby_parse_extract_error", file
+end
+
+task :bugs do
+  sh "for f in bug*.rb ; do rake19 debug F=$f && rm $f ; done"
+end
+
+# vim: syntax=Ruby

data/bin/ruby_parse

@@ -0,0 +1,96 @@
+#!/usr/bin/ruby -s
+
+$q ||= false
+$g ||= false
+
+require 'rubygems'
+require 'ruby_parser'
+require 'pp'
+
+good = bad = 0
+
+multi = ARGV.size != 1
+total_time = 0
+total_loc = 0
+total_kbytes = 0
+times = {}
+locs = {}
+kbytes = {}
+
+class File
+  RUBY19 = "<3".respond_to? :encoding
+
+  class << self
+    alias :binread :read unless RUBY19
+  end
+end
+
+begin
+  ARGV.each do |file|
+    rp = RubyParser.new
+    loc = `wc -l #{file}`.strip.to_i
+    size = `wc -c #{file}`.strip.to_i / 1024.0
+    locs[file] = loc
+    kbytes[file] = size
+    total_loc += loc
+    total_kbytes += size
+    if $q then
+      $stderr.print "."
+    else
+      warn "# file = #{file} loc = #{loc}"
+    end
+    GC.start if $g
+
+    t = Time.now
+    begin
+      begin
+        rp.reset
+        r = rp.parse(File.binread(file), file)
+        pp r unless $q
+        good += 1
+      rescue SyntaxError => e
+        warn "SyntaxError for #{file}: #{e.message}"
+        bad += 1
+      end
+    rescue => e
+      warn "#{e.backtrace.first} #{e.inspect.gsub(/\n/, ' ')} for #{file}"
+      warn "  #{e.backtrace.join("\n  ")}"
+      bad += 1
+    end
+
+    t = Time.now - t
+    times[file] = t
+    total_time += t
+  end
+rescue Interrupt
+  # do nothing
+end
+
+warn "done"
+
+total = 0
+times.values.each do |t|
+  total += t
+end
+
+puts
+puts "good = #{good} bad = #{bad}" if multi
+puts
+
+format = "%5.2fs:%9.2f l/s:%8.2f Kb/s:%5d Kb:%5d loc:%s"
+
+times.sort_by { |f, t| -t }.each do |f, t|
+  next if t < 0.005
+  loc = locs[f]
+  size = kbytes[f]
+  puts format % [t, loc / t, size / t, size, loc, f]
+end
+
+puts
+
+puts format % [total_time,
+               total_loc / total_time,
+               total_kbytes / total_time,
+               total_kbytes,
+               total_loc,
+               "TOTAL"] unless total_time == 0

data/bin/ruby_parse_extract_error

@@ -0,0 +1,130 @@
+#!/usr/bin/ruby -ws
+
+$d ||= false
+$d ||= ENV["DELETE"]
+$t ||= false
+$t ||= ENV["DELETE_TIMEOUT"]
+$m ||= false
+$m ||= ENV["MOVE_TIMEOUT"]
+$q ||= false
+$q ||= ENV["QUIET"]
+
+require 'rubygems'
+require 'ruby_parser'
+require 'fileutils'
+
+ARGV.push "-" if ARGV.empty?
+
+class Racc::Parser
+  def extract_defs
+    ss = lexer.src
+
+    raise "can't access source. possible encoding issue" unless ss
+
+    src = ss.string
+    pre_error = src[0...ss.pos]
+
+    defs = pre_error.grep(/^ *(?:def|it)/)
+
+    raise "can't figure out where the bad code starts" unless defs.last
+
+    last_def_indent = defs.last[/^ */]
+
+    post_error = src[ss.pos..-1]
+    idx = post_error =~ /^#{last_def_indent}end.*/
+
+    raise "can't figure out where the bad code ends" unless idx
+
+    src = pre_error + post_error[0..idx+$&.length]
+
+    src.scan(/^(( *)(?:def|it) .*?^\2end)/m)
+  end
+
+  def retest_for_errors defs
+    parser = self.class.new
+
+    parser.process(defs.join("\n\n"))
+  rescue SyntaxError, StandardError
+    nil
+  end
+end
+
+def expand path
+  if File.directory? path then
+    require 'find'
+
+    files = []
+
+    Find.find(*Dir[path]) do |f|
+      files << f if File.file? f
+    end
+
+    files.sort
+  else
+    Dir.glob path
+  end
+end
+
+def process_error parser
+  defs = parser.extract_defs
+
+  if parser.retest_for_errors defs then
+    warn "Can't reproduce error with just methods, punting..."
+    return
+  end
+
+  catch :extract_done do
+    (1..defs.size).each do |perm_size|
+      defs.combination(perm_size).each do |trial|
+        unless parser.retest_for_errors trial then
+          puts trial.join "\n"
+          throw :extract_done
+        end
+      end
+    end
+  end
+rescue RuntimeError, Racc::ParseError => e
+  warn "# process error: #{e.message.strip}"
+end
+
+def process file
+  ruby = file == "-" ? $stdin.read : File.read(file)
+  time = (ENV["RP_TIMEOUT"] || 10).to_i
+
+  $stderr.print "# Validating #{file}: "
+  parser = Ruby19Parser.new
+  parser.process(ruby, file, time)
+  warn "good"
+  File.unlink file if $d
+rescue Timeout::Error
+  $exit = 1
+  warn "TIMEOUT parsing #{file}. Skipping."
+
+  if $m then
+    dir = File.join $m, File.dirname(file)
+    FileUtils.mkdir_p dir
+    FileUtils.move file, dir
+  elsif $t then
+    File.unlink file
+  end
+rescue StandardError, SyntaxError, Racc::ParseError => e
+  $exit = 1
+  warn ""
+  warn "# error: #{e.message.strip}" unless $q
+  warn ""
+  return if $q
+
+  process_error parser
+end
+
+$exit = 0
+$stdout.sync = true
+
+ARGV.each do |path|
+  expand(path).each do |file|
+    next unless File.file? file # omg... why would you name a dir support.rb?
+    process file
+  end
+end
+
+exit $exit