parser 0.9.alpha → 0.9.alpha1

data/Manifest.txt

@@ -1,18 +0,0 @@
-.autotest
-History.txt
-Manifest.txt
-README.txt
-Rakefile
-bin/ruby_parse
-bin/ruby_parse_extract_error
-lib/gauntlet_rubyparser.rb
-lib/ruby18_parser.rb
-lib/ruby18_parser.y
-lib/ruby19_parser.rb
-lib/ruby19_parser.y
-lib/ruby_lexer.rb
-lib/ruby_parser.rb
-lib/ruby_parser_extras.rb
-test/test_ruby_lexer.rb
-test/test_ruby_parser.rb
-test/test_ruby_parser_extras.rb

data/README.txt

@@ -1,87 +0,0 @@
-= ruby_parser
-
-home :: https://github.com/seattlerb/ruby_parser
-bugs :: https://github.com/seattlerb/ruby_parser/issues
-rdoc :: http://docs.seattlerb.org/ruby_parser
-
-== DESCRIPTION:
-
-ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
-racc--which does by default use a C extension). RP's output is
-the same as ParseTree's output: s-expressions using ruby's arrays and
-base types.
-
-As an example:
-
-  def conditional1 arg1
-    return 1 if arg1 == 0
-    return 0
-  end
-
-becomes:
-
-  s(:defn, :conditional1, s(:args, :arg1),
-    s(:if,
-      s(:call, s(:lvar, :arg1), :==, s(:lit, 0)),
-      s(:return, s(:lit, 1)),
-      nil),
-    s(:return, s(:lit, 0)))
-
-== FEATURES/PROBLEMS:
-
-* Pure ruby, no compiles.
-* Includes preceding comment data for defn/defs/class/module nodes!
-* Incredibly simple interface.
-* Output is 100% equivalent to ParseTree.
-  * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
-* Known Issue: Speed is now pretty good, but can always improve:
-  * RP parses a corpus of 3702 files in 125s (avg 108 Kb/s)
-  * MRI+PT parsed the same in 67.38s (avg 200.89 Kb/s)
-* Known Issue: Code is much better, but still has a long way to go.
-* Known Issue: Totally awesome.
-* Known Issue: line number values can be slightly off. Parsing LR sucks.
-
-== SYNOPSIS:
-
-  RubyParser.new.parse "1+1"
-  # => s(:call, s(:lit, 1), :+, s(:lit, 1))
-
-You can also use Ruby19Parser, Ruby18Parser, or RubyParser.for_current_ruby:
-
-  RubyParser.for_current_ruby.parse "1+1"
-  # => s(:call, s(:lit, 1), :+, s(:lit, 1))
-
-== REQUIREMENTS:
-
-* ruby. woot.
-* sexp_processor for Sexp and SexpProcessor classes, and testing.
-* racc full package for parser development (compiling .y to .rb).
-
-== INSTALL:
-
-* sudo gem install ruby_parser
-
-== LICENSE:
-
-(The MIT License)
-
-Copyright (c) Ryan Davis, seattle.rb
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/ruby_parse

@@ -1,96 +0,0 @@
-#!/usr/bin/ruby -s
-
-$q ||= false
-$g ||= false
-
-require 'rubygems'
-require 'ruby_parser'
-require 'pp'
-
-good = bad = 0
-
-multi = ARGV.size != 1
-total_time = 0
-total_loc = 0
-total_kbytes = 0
-times = {}
-locs = {}
-kbytes = {}
-
-class File
-  RUBY19 = "<3".respond_to? :encoding
-
-  class << self
-    alias :binread :read unless RUBY19
-  end
-end
-
-begin
-  ARGV.each do |file|
-    rp = RubyParser.new
-    loc = `wc -l #{file}`.strip.to_i
-    size = `wc -c #{file}`.strip.to_i / 1024.0
-    locs[file] = loc
-    kbytes[file] = size
-    total_loc += loc
-    total_kbytes += size
-    if $q then
-      $stderr.print "."
-    else
-      warn "# file = #{file} loc = #{loc}"
-    end
-    GC.start if $g
-
-    t = Time.now
-    begin
-      begin
-        rp.reset
-        r = rp.parse(File.binread(file), file)
-        pp r unless $q
-        good += 1
-      rescue SyntaxError => e
-        warn "SyntaxError for #{file}: #{e.message}"
-        bad += 1
-      end
-    rescue => e
-      warn "#{e.backtrace.first} #{e.inspect.gsub(/\n/, ' ')} for #{file}"
-      warn "  #{e.backtrace.join("\n  ")}"
-      bad += 1
-    end
-
-    t = Time.now - t
-    times[file] = t
-    total_time += t
-  end
-rescue Interrupt
-  # do nothing
-end
-
-warn "done"
-
-total = 0
-times.values.each do |t|
-  total += t
-end
-
-puts
-puts "good = #{good} bad = #{bad}" if multi
-puts
-
-format = "%5.2fs:%9.2f l/s:%8.2f Kb/s:%5d Kb:%5d loc:%s"
-
-times.sort_by { |f, t| -t }.each do |f, t|
-  next if t < 0.005
-  loc = locs[f]
-  size = kbytes[f]
-  puts format % [t, loc / t, size / t, size, loc, f]
-end
-
-puts
-
-puts format % [total_time,
-               total_loc / total_time,
-               total_kbytes / total_time,
-               total_kbytes,
-               total_loc,
-               "TOTAL"] unless total_time == 0

data/bin/ruby_parse_extract_error

@@ -1,130 +0,0 @@
-#!/usr/bin/ruby -ws
-
-$d ||= false
-$d ||= ENV["DELETE"]
-$t ||= false
-$t ||= ENV["DELETE_TIMEOUT"]
-$m ||= false
-$m ||= ENV["MOVE_TIMEOUT"]
-$q ||= false
-$q ||= ENV["QUIET"]
-
-require 'rubygems'
-require 'ruby_parser'
-require 'fileutils'
-
-ARGV.push "-" if ARGV.empty?
-
-class Racc::Parser
-  def extract_defs
-    ss = lexer.src
-
-    raise "can't access source. possible encoding issue" unless ss
-
-    src = ss.string
-    pre_error = src[0...ss.pos]
-
-    defs = pre_error.grep(/^ *(?:def|it)/)
-
-    raise "can't figure out where the bad code starts" unless defs.last
-
-    last_def_indent = defs.last[/^ */]
-
-    post_error = src[ss.pos..-1]
-    idx = post_error =~ /^#{last_def_indent}end.*/
-
-    raise "can't figure out where the bad code ends" unless idx
-
-    src = pre_error + post_error[0..idx+$&.length]
-
-    src.scan(/^(( *)(?:def|it) .*?^\2end)/m)
-  end
-
-  def retest_for_errors defs
-    parser = self.class.new
-
-    parser.process(defs.join("\n\n"))
-  rescue SyntaxError, StandardError
-    nil
-  end
-end
-
-def expand path
-  if File.directory? path then
-    require 'find'
-
-    files = []
-
-    Find.find(*Dir[path]) do |f|
-      files << f if File.file? f
-    end
-
-    files.sort
-  else
-    Dir.glob path
-  end
-end
-
-def process_error parser
-  defs = parser.extract_defs
-
-  if parser.retest_for_errors defs then
-    warn "Can't reproduce error with just methods, punting..."
-    return
-  end
-
-  catch :extract_done do
-    (1..defs.size).each do |perm_size|
-      defs.combination(perm_size).each do |trial|
-        unless parser.retest_for_errors trial then
-          puts trial.join "\n"
-          throw :extract_done
-        end
-      end
-    end
-  end
-rescue RuntimeError, Racc::ParseError => e
-  warn "# process error: #{e.message.strip}"
-end
-
-def process file
-  ruby = file == "-" ? $stdin.read : File.read(file)
-  time = (ENV["RP_TIMEOUT"] || 10).to_i
-
-  $stderr.print "# Validating #{file}: "
-  parser = Ruby19Parser.new
-  parser.process(ruby, file, time)
-  warn "good"
-  File.unlink file if $d
-rescue Timeout::Error
-  $exit = 1
-  warn "TIMEOUT parsing #{file}. Skipping."
-
-  if $m then
-    dir = File.join $m, File.dirname(file)
-    FileUtils.mkdir_p dir
-    FileUtils.move file, dir
-  elsif $t then
-    File.unlink file
-  end
-rescue StandardError, SyntaxError, Racc::ParseError => e
-  $exit = 1
-  warn ""
-  warn "# error: #{e.message.strip}" unless $q
-  warn ""
-  return if $q
-
-  process_error parser
-end
-
-$exit = 0
-$stdout.sync = true
-
-ARGV.each do |path|
-  expand(path).each do |file|
-    next unless File.file? file # omg... why would you name a dir support.rb?
-    process file
-  end
-end
-
-exit $exit

data/lib/gauntlet_rubyparser.rb

@@ -1,117 +0,0 @@
-#!/usr/bin/ruby -ws
-
-$f ||= false
-
-$:.unshift "../../ruby_parser/dev/lib"
-$:.unshift "../../ruby2ruby/dev/lib"
-
-require 'rubygems'
-require 'ruby2ruby'
-require 'ruby_parser'
-
-require 'gauntlet'
-
-class RubyParserGauntlet < Gauntlet
-  def initialize
-    super
-
-    self.data = Hash.new { |h,k| h[k] = {} }
-    old_data = load_yaml data_file
-    self.data.merge! old_data
-  end
-
-  def should_skip? name
-    if $f then
-      if Hash === data[name] then
-        ! data[name].empty?
-      else
-        data[name]
-      end
-    else
-      data[name] == true # yes, == true on purpose
-    end
-  end
-
-  def diff_pp o1, o2
-    require 'pp'
-
-    Tempfile.new('ruby_parser_a') do |file_a|
-      PP.pp o1, file_a
-
-      Tempfile.new('ruby_parser_b') do |file_b|
-        PP.pp o2, file_b
-
-        `diff -u #{file_a.path} #{file_b.path}`
-      end
-    end
-  end
-
-  def broke name, file, msg
-    warn "bad"
-    self.data[name][file] = msg
-    self.dirty = true
-  end
-
-  def process path, name
-    begin
-      $stderr.print "  #{path}: "
-      rp = RubyParser.new
-      r2r = Ruby2Ruby.new
-
-      old_ruby = File.read(path)
-
-      begin
-        old_sexp = rp.process old_ruby
-      rescue Racc::ParseError => e
-        self.data[name][path] = :unparsable
-        self.dirty = true
-        return
-      end
-
-      new_ruby = r2r.process old_sexp.deep_clone
-
-      begin
-        new_sexp = rp.process new_ruby
-      rescue Racc::ParseError => e
-        broke name, path, "couldn't parse new_ruby: #{e.message.strip}"
-        return
-      end
-
-      if old_sexp != new_sexp then
-        broke name, path, diff_pp(old_sexp, new_sexp)
-        return
-      end
-
-      self.data[name][path] = true
-      self.dirty = true
-
-      warn "good"
-    rescue Interrupt
-      puts "User cancelled"
-      exit 1
-    rescue Exception => e
-      broke name, path, "    UNKNOWN ERROR: #{e}: #{e.message.strip}"
-    end
-  end
-
-  def run name
-    warn name
-    Dir["**/*.rb"].sort.each do |path|
-      next if path =~ /gemspec.rb/ # HACK
-      next if data[name][path] == true
-      process path, name
-    end
-
-    if self.data[name].values.all? { |v| v == true } then
-      warn "  ALL GOOD!"
-      self.data[name] = true
-      self.dirty = true
-    end
-  end
-end
-
-filter = ARGV.shift
-filter = Regexp.new filter if filter
-
-gauntlet = RubyParserGauntlet.new
-gauntlet.run_the_gauntlet filter