ruby_parser 2.0.5 → 3.0.0.a1

data/.autotest

@@ -1,11 +1,12 @@
 # -*- ruby -*-
 
 require 'autotest/restart'
+require 'autotest/isolate'
 require 'autotest/rcov' if ENV['RCOV']
 
 Autotest.add_hook :initialize do |at|
-  at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
-  at.libs << ":../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
+  at.libs << ":../../sexp_processor/dev/lib"
   at.add_exception 'unit'
   at.add_exception 'coverage'
   at.add_exception 'coverage.info'
@@ -13,7 +14,6 @@ Autotest.add_hook :initialize do |at|
 
   at.libs << ':../../minitest/dev/lib'
   at.testlib = "minitest/autorun"
-  at.unit_diff = "unit_diff -u -b"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
     at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
@@ -27,7 +27,7 @@ Autotest.add_hook :initialize do |at|
     at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
   end
 
-  %w(TestRubyParser TestParseTree).each do |klass| # HACK
+  %w(TestRuby18Parser TestRuby19Parser TestParseTree).each do |klass| # HACK
     at.extra_class_map[klass] = "test/test_ruby_parser.rb"
   end
 end

data/History.txt

@@ -1,3 +1,127 @@
+=== 3.0.0.a1 / 2012-05-22
+
+This is the first alpha release of the 3.0.0 series. It is probably
+woefully incomplete, bug ridden, and hasn't showered in several days.
+Please please please beat the crap out of it and send
+bugs/patches/complaints/suggestions.
+
+* 5 major enhancements:
+
+  * 1.9 parsing! Thanks to EVERYONE for submitting patches for this!
+  * Removed :arglist from everything but :op_asgn1
+  * Removed :block from resbody
+  * Removed :block from when
+  * Removed :block nodes inside of scope nodes (defn/defs/class/sclass).
+  * Removed :scope nodes in defn/defs/class/sclass nodes.
+  * (probably more sexp cleanup to come before 3.0.0 final)
+
+* 25 minor enhancements:
+
+  * 1.9: Fix \!a. (wanabe)
+  * 1.9: Method calling with postargs. (wanabe)
+  * 1.9: Method definition with postargs. (wanabe)
+  * 1.9: Support lambda args without parentheses. (wanabe)
+  * Added R arg to `rake debug` to debug ruby straight up
+  * Added RubyParser, subclassing Ruby18Parser but warning on instantiation.
+  * Added backref_assign_error (needs tests)
+  * Added bin/ruby_parse_extract_error to help with error submissions
+  * Added debug task to help quickly get into a bug
+  * Added more 18 vs 19 lexing tests for ?c.
+  * Added ruby_parser.rb that pulls everything together in proper order.
+  * Added tLABEL. (brynary)
+  * Branched ruby_parser.y to ruby18_parser.y
+  * Fix to pass test_lambda_args_block__19 test. (mrmargolis)
+  * Got rid of one instance of unread_many. (Confusion)
+  * Moved everything from RubyParser to RubyParserStuff and included module in both.
+  * Refactored 1.9 args handling
+  * Refactored and added new_resbody to ruby_parser_extras.
+  * Refactored and added new_when
+  * Refactored tests infrastructure and added both 1.8 and 1.9 test branches.
+  * Removed unused methods: unread, begin_of_line? was_begin_of_line. (YAY!) (Confusion)
+  * Renamed ruby_parser.y to ruby19_parser.y
+  * RubyLexer now takes a version specifier.
+  * Started doing comparative refactoring between MRI's 1.9 parser and RP's. Shouldn't differ functionally except where we missed stuff in RP.
+  * `rake debug` prints the output if it succeeds.
+
+* 21 bug fixes:
+
+  * Added missing gvar arg error. (1.8)
+  * Attach parser files to isolate to ensure they can build
+  * Conditionalize handling of tLABEL to ruby19+. Fixes issue #33.
+  * DOH. I deactivated some tests and never reactivated them. (Confusion ftw)
+  * Duplicate the input so that heredoc processing doesn't morph original. (banister)
+  * Entirely reworked block arg handling. (1.8)
+  * Fix ?x char literal. (nobu)
+  * Fixed 4/5 of literal lambda tests (jamie)
+  * Fixed deps for parser
+  * Fixed lexing of ?c for ruby 1.8 and 1.9.
+  * Fixed more Ruby 1.9 args tests (brynary)
+  * Fixed reswords to match MRI (1.8, 1.9)
+  * Fixed symbols with no spaces in method calls (e.g. foo:bar) (YAY! brynary)
+  * Fixed ternary_nil_no_space and other ternary edge cases for 1.9. (lastobelus)
+  * Fixed test_call_not_equal__19. First bug bounty! (albus522)
+  * Made lambda w/o arg list zero out the arg slot.
+  * Renamed awords to qwords to match stupid MRI naming. (1.8, 1.9) :(
+  * Rolled out brynary's symbols-no-spaces (foo:bar) changes when parsing 1.8 code
+  * Split 1.8 from 1.9 open paren lexer. Gawd that's ugly code.
+  * Split block_var from for_var. (1.8, 1.9)
+  * Use binread (and emulate in ruby 1.8) to avoid encoding issues
+
+=== 2.3.1 / 2011-09-21
+
+* 2 bug fixes:
+
+  * Fixed line numbers at end of special var+whitespace (larsch)
+  * Holy crap I was smokin' something good... Fixed 1.9.3 warning
+
+=== 2.3.0 / 2011-09-06
+
+* 2 minor enhancements:
+
+  * Add -g flag to parser compile if DEBUG
+  * Lexer now embeds line number in yacc_value for keywords, helping fix up line numbers
+
+* 3 bug fixes:
+
+  * Fix method line numbers when no args and no parens (quix)
+  * Fixed line numbers on return/break/next w/ result expr. (pjnz)
+  * Fixed some lexing state in order to parse: 'f (1), 2' as 'f(1, 2)'. (invernizzi)
+
+=== 2.2.0 / 2011-08-23
+
+* 2 minor enhancements:
+
+  * Moved Keyword, Environment, and StackState inside of RubyParser
+  * Added proper dsym and dsym->sym support.
+
+* 3 bug fixes:
+
+  * Added extra (failing) tests for call/iter line number checking (quix)
+  * Fixed line numbers for certain call/iter edge cases
+  * Fixed parsing of: alias :"<<" :">>".
+
+=== 2.1.0 / 2011-08-15
+
+* 2 minor enhancements:
+
+  * Added new accessor canonicalize_conditions to toggle conditional canonicalization (on by default). (confused)
+  * Awesome cleanup: Replaced call to append_block by block_append. (Confusion)
+
+* 2 bug fixes:
+
+  * Fixed handling last line of =begin/=end. (raybaxter)
+  * Fixed source line numbers after heredocs. (jbarreneche)
+
+=== 2.0.6 / 2011-02-18
+
+* 1 minor enhancement:
+
+  * Switched to hoe's racc plugin to clean up rakefile and builds
+
+* 1 bug fix:
+
+  * Fixed empty =begin/end.
+
 === 2.0.5 / 2010-09-01
 
 * 1 minor enhancement:

data/Manifest.txt

@@ -4,9 +4,14 @@ Manifest.txt
 README.txt
 Rakefile
 bin/ruby_parse
+bin/ruby_parse_extract_error
 lib/gauntlet_rubyparser.rb
+lib/ruby18_parser.rb
+lib/ruby18_parser.y
+lib/ruby19_parser.rb
+lib/ruby19_parser.y
 lib/ruby_lexer.rb
-lib/ruby_parser.y
+lib/ruby_parser.rb
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb

data/README.txt

@@ -1,6 +1,8 @@
 = ruby_parser
 
-* http://parsetree.rubyforge.org/
+home :: https://github.com/seattlerb/ruby_parser
+bugs :: https://github.com/seattlerb/ruby_parser/issues
+rdoc :: http://docs.seattlerb.org/ruby_parser
 
 == DESCRIPTION:
 
@@ -46,14 +48,13 @@ becomes:
 
 == SYNOPSIS:
 
-  RubyParser.new.parse "1+1"
-  # => s(:call, s(:lit, 1), :+, s(:array, s(:lit, 1)))
+  Ruby19Parser.new.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:lit, 1))
 
 == REQUIREMENTS:
 
 * ruby. woot.
-* sexp_processor for Sexp and SexpProcessor classes.
-* ParseTree for testing.
+* sexp_processor for Sexp and SexpProcessor classes, and testing.
 * racc full package for parser development (compiling .y to .rb).
 
 == INSTALL:
@@ -64,7 +65,7 @@ becomes:
 
 (The MIT License)
 
-Copyright (c) 2007-2008 Ryan Davis
+Copyright (c) Ryan Davis, seattle.rb
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -4,49 +4,34 @@ require 'rubygems'
 require 'hoe'
 
 Hoe.plugin :seattlerb
+Hoe.plugin :racc
+Hoe.plugin :isolate
 
-Hoe.add_include_dirs("../../ParseTree/dev/test",
-                     "../../RubyInline/dev/lib",
-                     "../../sexp_processor/dev/lib")
+Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 
-hoe = Hoe.spec 'ruby_parser' do
+Hoe.spec 'ruby_parser' do
   developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
 
   self.rubyforge_name = 'parsetree'
 
-  extra_dev_deps << ['ParseTree',      '~> 3.0']
-  extra_deps     << ['sexp_processor', '~> 3.0']
-end
-
-hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
+  dependency 'sexp_processor', '~> 3.0'
 
-[:default, :multi, :test].each do |t|
-  task t => :parser
-end
-
-path = "pkg/ruby_parser-#{hoe.version}"
-task path => :parser do
-  Dir.chdir path do
-    sh "rake parser"
+  if plugin? :perforce then
+    self.perforce_ignore << "lib/ruby18_parser.rb"
+    self.perforce_ignore << "lib/ruby19_parser.rb"
   end
-end
 
-desc "build the parser"
-task :parser => ["lib/ruby_parser.rb"]
-
-rule '.rb' => '.y' do |t|
-  # -v = verbose
-  # -t = debugging parser ~4% reduction in speed -- keep for now
-  # -l = no-line-convert
-  sh "racc -v -t -l -o #{t.name} #{t.source}"
+  self.racc_flags << " -g" if plugin?(:racc) && ENV["DEBUG"]
 end
 
+file "lib/ruby18_parser.rb" => "lib/ruby18_parser.y"
+file "lib/ruby19_parser.rb" => "lib/ruby19_parser.y"
+
 task :clean do
   rm_rf(Dir["**/*~"] +
         Dir["**/*.diff"] +
         Dir["coverage.info"] +
         Dir["coverage"] +
-        Dir["lib/ruby_parser.rb"] +
         Dir["lib/*.output"])
 end
 
@@ -124,4 +109,66 @@ task :huh? do
   puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
 end
 
+task :irb => [:isolate] do
+  sh "GEM_HOME=#{Gem.path.first} irb -rubygems -Ilib -rruby_parser;"
+end
+
+def (task(:phony)).timestamp
+  Time.at 0
+end
+
+task :isolate => :phony
+
+file "lib/ruby18_parser.rb" => :isolate
+file "lib/ruby19_parser.rb" => :isolate
+
+task :compare18 do
+  sh "./yack.rb lib/ruby18_parser.output > racc18.txt"
+  sh "./yack.rb parse18.output > yacc18.txt"
+  sh "diff -du racc18.txt yacc18.txt || true"
+  puts
+  sh "diff -du racc18.txt yacc18.txt | wc -l"
+end
+
+task :compare19 do
+  sh "./yack.rb lib/ruby19_parser.output > racc19.txt"
+  sh "./yack.rb parse19.output > yacc19.txt"
+  sh "diff -du racc19.txt yacc19.txt || true"
+  puts
+  sh "diff -du racc19.txt yacc19.txt | wc -l"
+end
+
+task :debug => :isolate do
+  ENV["V"] ||= "18"
+  Rake.application[:parser].invoke # this way we can have DEBUG set
+
+  $: << "lib"
+  require 'ruby_parser'
+
+  parser = if ENV["V"] == "18" then
+             Ruby18Parser.new
+           else
+             Ruby19Parser.new
+           end
+
+  file = ENV["F"] || ENV["FILE"]
+
+  ruby = if file then
+           File.read(file)
+         else
+           file = "env"
+           ENV["R"] || ENV["RUBY"]
+         end
+
+  begin
+    p parser.process(ruby, file)
+  rescue Racc::ParseError => e
+    p e
+    ss = parser.lexer.src
+    src = ss.string
+    lines = src[0..ss.pos].split(/\n/)
+    abort "on #{file}:#{lines.size}"
+  end
+end
+
 # vim: syntax=Ruby

data/bin/ruby_parse

@@ -17,6 +17,14 @@ times = {}
 locs = {}
 kbytes = {}
 
+class File
+  RUBY19 = "<3".respond_to? :encoding
+
+  class << self
+    alias :binread :read unless RUBY19
+  end
+end
+
 begin
   ARGV.each do |file|
     rp = RubyParser.new
@@ -37,7 +45,7 @@ begin
     begin
       begin
         rp.reset
-        r = rp.parse(File.read(file), file)
+        r = rp.parse(File.binread(file), file)
         pp r unless $q
         good += 1
       rescue SyntaxError => e

data/bin/ruby_parse_extract_error

@@ -0,0 +1,75 @@
+#!/usr/bin/ruby -ws
+
+require 'rubygems'
+require 'ruby_parser'
+
+ARGV.push "-" if ARGV.empty?
+
+parser = Ruby18Parser.new
+
+class Racc::Parser
+  def extract_defs
+    ss = lexer.src
+    src = ss.string
+    pre_error = src[0...ss.pos]
+
+    defs = pre_error.grep(/^ *def/)
+
+    last_def_indent = defs.last[/^ */]
+
+    post_error = src[ss.pos..-1]
+    idx = post_error =~ /^#{last_def_indent}end.*/
+
+    raise "can't figure out where the bad code ends" unless idx
+
+    src = pre_error + post_error[0..idx+$&.length]
+
+    src.scan(/^(( *)def .*?^\2end)/m)
+  end
+
+  def retest_for_errors defs
+    d :retest => defs.size
+
+    parser = self.class.new
+
+    parser.process(defs.join("\n\n")) rescue nil
+  end
+end
+
+ARGV.replace ARGV.map { |path|
+  if File.directory? path then
+    Dir.glob File.join path, "**/*.rb"
+  else
+    path
+  end
+}.flatten
+
+ARGV.each do |file|
+  ruby = file == "-" ? $stdin.read : File.read(file)
+
+  begin
+    $stderr.print "Validating #{file}: "
+    parser.process(ruby, file)
+    $stderr.puts "good"
+  rescue Racc::ParseError => e
+    $stderr.puts "error: #{e.description}"
+    $stderr.puts
+
+    defs = parser.extract_defs
+
+    abort "Can't reproduce error with just methods, punting..." if
+      parser.retest_for_errors defs
+
+    mandatory = defs.pop
+
+    (1..defs.size).each do |perm_size|
+      defs.combination(perm_size).each do |trial|
+        trial << mandatory
+        unless parser.retest_for_errors trial then
+          puts trial.join "\n"
+          abort "reduced repro found!"
+        end
+      end
+    end
+  end
+end