ParseTree 2.2.0 → 3.0.0

data/.autotest

@@ -0,0 +1,16 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+Autotest.add_hook :initialize do |at|
+  at.libs << ":../../sexp_processor/dev/lib"
+  at.extra_files << "test/pt_testcase.rb"
+
+  %w(TestRawParseTree).each do |klass|
+    at.extra_class_map[klass] = "test/test_parse_tree.rb"
+  end
+
+  at.add_mapping(/pt_testcase/) do |f, _|
+    at.files_matching(/test_.*rb$/)
+  end
+end

data/History.txt

@@ -1,3 +1,51 @@
+=== 3.0.0 / 2008-10-22
+
+* 3 major enhancements:
+
+  * Split out sexp_processor project.
+  * ParseTree#process outputs a UnifiedRuby parse tree.
+    * Switched all ParseTree tests to RawParseTree tests.
+    * Added ParseTree tests.
+  * Added UnifiedRuby#process.
+
+* 28 minor enhancements:
+
+  * PTTC: Added a comprehensive suite of tests for every combo of defn args.
+  * PTTC: Added a ton of new tests.
+  * PTTC: Added extra tests for mri verbose flag testing (horrid horrid bug imhfo)
+  * PTTC: Don't generate tests if class has "TestCase" in the class name.
+  * PTTC: Made self.previous more overridable.
+  * PTTC: Now all tests that have "mri_verbose_flag" activate $VERBOSE
+  * PTTC: Officially added clone_same phase to generate_tests
+  * PTTC: Refactored test generation to make subclasses more flexible.
+  * PTTC: Refactored to use new add_tests. Much much cleaner.
+  * PTTC: Removed Unique from PTTC... subclasses need to deal with this
+  * PTTC: Removed some stupid tests.
+  * PTTC: Renamed a bunch of tests to be more uniform.
+  * UR: Added rewrite_attrasgn to normalize arg array into arglist.
+  * UR: Added rewrite_op_asgn1 to ensure the arglist is an arglist.
+  * UR: Flattened resbodies in rescue node.
+  * UR: Flattened super and yield args yet, deal with array arg properly.
+  * UR: Handle yield(*ary) properly.
+  * UR: Nuked argspush.
+  * UR: Removed begin node from unified sexps.
+  * UR: Removed block from rewritten resbody if only 1 element.
+  * UR: Removed dasgn, dasgn_curr, and dvar nodes.
+  * UR: Removed s(:block_arg, :block) in favor of :"&block"
+  * UR: Rewrote argscat to be a splat inside an array.
+  * UR: Rewrote block_pass into arglist of inner call/super.
+  * UR: Rewrote call to ALWAYS have an argslist (no more splat as arglist).
+  * UR: Rewrote rewrite_resbody from scratch... MUUCH cleaner.
+  * UR: zarray -> array
+  * Used Hoe#add_include_dirs to clean up rakefile.
+  * Moved PT dependent code from ruby2ruby to here (eg Proc#to_sexp & friends).
+
+* 2 bug fixes:
+
+  * UR: Fixed a bug with resbody that starts with an lasgn.
+  * UR: Fixed args for attrset
+  * Fixed export of symbols to work across platforms.
+
 === 2.2.0 / 2008-06-09
 
 * 18 minor enhancements:

data/Manifest.txt

@@ -1,3 +1,4 @@
+.autotest
 History.txt
 Manifest.txt
 README.txt
@@ -7,18 +8,14 @@ bin/parse_tree_audit
 bin/parse_tree_deps
 bin/parse_tree_show
 demo/printer.rb
-lib/composite_sexp_processor.rb
 lib/parse_tree.rb
-lib/sexp.rb
-lib/sexp_processor.rb
+lib/parse_tree_extensions.rb
 lib/unified_ruby.rb
 lib/unique.rb
 test/pt_testcase.rb
 test/something.rb
 test/test_all.rb
-test/test_composite_sexp_processor.rb
 test/test_parse_tree.rb
-test/test_sexp.rb
-test/test_sexp_processor.rb
+test/test_parse_tree_extensions.rb
 test/test_unified_ruby.rb
 validate.sh

data/Rakefile

@@ -3,7 +3,11 @@
 require 'rubygems'
 require 'hoe'
 
-$: << "../../RubyInline/dev/lib"
+Hoe.add_include_dirs("../../RubyInline/dev/lib",
+                     "../../sexp_processor/dev/lib",
+                     "../../ZenTest/dev/lib",
+                     "lib")
+
 require './lib/parse_tree.rb'
 
 Hoe.new("ParseTree", ParseTree::VERSION) do |pt|
@@ -12,14 +16,13 @@ Hoe.new("ParseTree", ParseTree::VERSION) do |pt|
   pt.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
 
   pt.clean_globs << File.expand_path("~/.ruby_inline")
-  pt.extra_deps << ['RubyInline', '>= 3.6.0']
+  pt.extra_deps << ['RubyInline',     '>= 3.7.0']
+  pt.extra_deps << ['SexpProcessor', '>= 3.0.0']
   pt.spec_extras[:require_paths] = proc { |paths| paths << 'test' }
 
-  pt.multiruby_skip << "1.9" << "rubinius"
+  pt.multiruby_skip << "mri_rel_1_9" << "rubinius" << "mri_trunk"
 end
 
-Hoe::RUBY_FLAGS.sub! /-I/, '-I../../RubyInline/dev/lib:test:'
-
 desc 'Run in gdb'
 task :debug do
   puts "RUN: r -d #{Hoe::RUBY_FLAGS} test/test_all.rb #{Hoe::FILTER}"

data/lib/parse_tree.rb

@@ -7,6 +7,7 @@ raise LoadError, "ParseTree isn't needed with rubinius" if
 
 require 'rubygems'
 require 'inline'
+require 'unified_ruby'
 
 class Module
   def modules
@@ -40,9 +41,9 @@ end
 #                [:args],
 #                [:return, [:call, [:lit, 1], "+", [:array, [:lit, 1]]]]]]]]]
 
-class ParseTree
+class RawParseTree
 
-  VERSION = '2.2.0'
+  VERSION = '3.0.0'
 
   ##
   # Front end translation method.
@@ -288,7 +289,7 @@ class ParseTree
     # 1) Get me a login on your box so I can repro this and get it fixed.
     # 2) Fix it and send me the patch
     # 3) (quick, but dirty and bad), comment out the following line:
-    builder.add_compile_flags "-Werror"
+    builder.add_compile_flags "-Werror" unless RUBY_PLATFORM =~ /mswin/
 
     builder.prefix %{
         #define nd_3rd   u3.node
@@ -350,7 +351,7 @@ void add_to_parse_tree(VALUE self, VALUE ary, NODE * n, ID * locals) {
   static int masgn_level = 0;
 
   if (NIL_P(node_names)) {
-    node_names = rb_const_get_at(rb_path2class("ParseTree"),rb_intern("NODE_NAMES"));
+    node_names = rb_const_get_at(rb_path2class("RawParseTree"),rb_intern("NODE_NAMES"));
   }
 
   if (!node) return;
@@ -508,15 +509,27 @@ again:
 
   case NODE_BREAK:
   case NODE_NEXT:
+    if (node->nd_stts)
+      add_to_parse_tree(self, current, node->nd_stts, locals);
+
+    break;
+
   case NODE_YIELD:
     if (node->nd_stts)
       add_to_parse_tree(self, current, node->nd_stts, locals);
+
+    if (node->nd_stts
+        && (nd_type(node->nd_stts) == NODE_ARRAY
+            || nd_type(node->nd_stts) == NODE_ZARRAY)
+        && !node->nd_state)
+      rb_ary_push(current, Qtrue);
+
     break;
 
   case NODE_RESCUE:
-      add_to_parse_tree(self, current, node->nd_1st, locals);
-      add_to_parse_tree(self, current, node->nd_2nd, locals);
-      add_to_parse_tree(self, current, node->nd_3rd, locals);
+    add_to_parse_tree(self, current, node->nd_1st, locals);
+    add_to_parse_tree(self, current, node->nd_2nd, locals);
+    add_to_parse_tree(self, current, node->nd_3rd, locals);
     break;
 
   /*
@@ -665,15 +678,25 @@ again:
 
   case NODE_MASGN:
     masgn_level++;
-    add_to_parse_tree(self, current, node->nd_head, locals);
+    if (node->nd_head) {
+      add_to_parse_tree(self, current, node->nd_head, locals);
+    } else {
+      rb_ary_push(current, Qnil);
+    }
     if (node->nd_args) {
       if (node->nd_args != (NODE *)-1) {
         add_to_parse_tree(self, current, node->nd_args, locals);
       } else {
         rb_ary_push(current, wrap_into_node("splat", 0));
       }
+    } else {
+      rb_ary_push(current, Qnil);
+    }
+    if (node->nd_value) {
+      add_to_parse_tree(self, current, node->nd_value, locals);
+    } else {
+      rb_ary_push(current, Qnil);
     }
-    add_to_parse_tree(self, current, node->nd_value, locals);
     masgn_level--;
     break;
 
@@ -1051,9 +1074,12 @@ static VALUE parse_tree_for_meth(VALUE klass, VALUE method, VALUE is_cls_meth) {
 }
 }
 
-    builder.prefix " extern NODE *ruby_eval_tree_begin; " \
+    extern_mode = RUBY_PLATFORM =~ /mswin/ ? 'RUBY_EXTERN' : 'extern'
+    builder.prefix " #{extern_mode} NODE *ruby_eval_tree_begin; " \
       if RUBY_VERSION < '1.9.0'
 
+    # FIXME: ruby_in_eval is not properly exported across platforms
+    # http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-core/13558
     builder.c %Q{
 static VALUE parse_tree_for_str(VALUE source, VALUE filename, VALUE line) {
   VALUE tmp;
@@ -1094,4 +1120,34 @@ static VALUE parse_tree_for_str(VALUE source, VALUE filename, VALUE line) {
 }
 
   end # inline call
-end # ParseTree class
+end # RawParseTree class
+
+class ParseTree < RawParseTree
+  ##
+  # Initializes a ParseTree instance. Includes newline nodes if
+  # +include_newlines+ which defaults to +$DEBUG+.
+
+  def initialize(include_newlines=$DEBUG)
+    super
+    @unifier = Unifier.new
+  end
+
+  ##
+  # Main driver for ParseTree. Returns a Sexp instance containing the
+  # AST representing the input given. This is a UnifiedRuby sexp, not
+  # a raw sexp from ruby. If you want raw, use the old
+  # parse_tree_for_xxx methods... Please tell me if/why you want raw,
+  # I'd like to know so I can justify keeping the code around.
+
+  def process(input, verbose = nil, file = "(string)", line = -1)
+    case input
+    when Array then
+      @unifier.process(input)
+    when String then
+      pt = self.parse_tree_for_string(input, file, line, verbose).first
+      @unifier.process(pt)
+    else
+      raise ArgumentError, "Unknown input type #{input.inspect}"
+    end
+  end
+end

data/lib/parse_tree_extensions.rb

@@ -0,0 +1,69 @@
+class Method
+  def with_class_and_method_name
+    if self.inspect =~ /<Method: (.*)\#(.*)>/ then
+      klass = eval $1
+      method  = $2.intern
+      raise "Couldn't determine class from #{self.inspect}" if klass.nil?
+      return yield(klass, method)
+    else
+      raise "Can't parse signature: #{self.inspect}"
+    end
+  end
+
+  def to_sexp
+    require 'parse_tree'
+    require 'unified_ruby'
+    parser = ParseTree.new(false)
+    unifier = Unifier.new
+    with_class_and_method_name do |klass, method|
+      old_sexp = parser.parse_tree_for_method(klass, method)
+      unifier.process(old_sexp) # HACK
+    end
+  end
+
+  def to_ruby
+    sexp = self.to_sexp
+    Ruby2Ruby.new.process sexp
+  end
+end
+
+class ProcStoreTmp
+  @@n = 0
+  def self.new_name
+    @@n += 1
+    return :"myproc#{@@n}"
+  end
+end
+
+class UnboundMethod
+  def to_ruby
+    name = ProcStoreTmp.new_name
+    ProcStoreTmp.send(:define_method, name, self)
+    m = ProcStoreTmp.new.method(name)
+    result = m.to_ruby.sub(/def #{name}(?:\(([^\)]*)\))?/,
+                           'proc { |\1|').sub(/end\Z/, '}')
+    return result
+  end
+end
+
+class Proc
+  def to_method
+    name = ProcStoreTmp.new_name
+    ProcStoreTmp.send(:define_method, name, self)
+    ProcStoreTmp.new.method(name)
+  end
+
+  def to_sexp
+    sexp = self.to_method.to_sexp
+    body = sexp.scope.block
+    args = sexp.args
+    args = nil if args.size == 1
+    body = body[1] if body.size == 2
+
+    s(:iter, s(:call, nil, :proc, s(:arglist)), args, body)
+  end
+
+  def to_ruby
+    Ruby2Ruby.new.process(self.to_sexp).sub(/^\Aproc do/, 'proc {').sub(/end\Z/, '}')
+  end
+end

data/lib/unified_ruby.rb

@@ -1,17 +1,37 @@
+require 'sexp_processor'
 
 $TESTING ||= false
 
 module UnifiedRuby
+  def process exp
+    exp = Sexp.from_array exp unless Sexp === exp or exp.nil?
+    super
+  end
+
+  def rewrite_attrasgn(exp)
+    last = exp.last
+
+    if Sexp === last then
+      last[0] = :arglist if last[0] == :array
+    else
+      exp << s(:arglist)
+    end
+
+    exp
+  end
+
   def rewrite_bmethod(exp)
     exp[0] = :scope
 
+    has_splat = exp.masgn.array.splat.lasgn rescue false
+
     args =
-      if exp.masgn and exp.masgn.dasgn_curr then
-        arg = exp.masgn(true).dasgn_curr(true).sexp_body
+      if has_splat then
+        arg = exp.masgn(true).array.splat.lasgn.sexp_body
         raise "nope: #{arg.size}" unless arg.size == 1
         s(:args, :"*#{arg.last}")
       else
-        args = exp.dasgn_curr(true)
+        args = exp.lasgn(true)
         if args then
           s(:args, *args.sexp_body)
         else
@@ -27,6 +47,35 @@ module UnifiedRuby
     exp
   end
 
+  def rewrite_argscat exp
+    _, ary, val = exp
+    ary = s(:array, ary) unless ary.first == :array
+    ary << s(:splat, val)
+  end
+
+  def rewrite_argspush exp
+    exp[0] = :arglist
+    exp
+  end
+
+  def rewrite_block_pass exp
+    if exp.size == 3 then
+      _, block, recv = exp
+      case recv.first
+      when :super then
+        recv << s(:block_pass, block)
+        exp = recv
+      when :call then
+        recv.last << s(:block_pass, block)
+        exp = recv
+      else
+        raise "huh?: #{recv.inspect}"
+      end
+    end
+
+    exp
+  end
+
   def rewrite_call(exp)
     args = exp.last
     case args
@@ -37,7 +86,7 @@ module UnifiedRuby
       when :array, :arglist then
         args[0] = :arglist
       when :argscat, :splat then
-        # do nothing
+        exp[-1] = s(:arglist, args)
       else
         raise "unknown type in call #{args.first.inspect} in #{exp.inspect}"
       end
@@ -49,6 +98,13 @@ module UnifiedRuby
     exp
   end
 
+  def rewrite_dasgn(exp)
+    exp[0] = :lasgn
+    exp
+  end
+
+  alias :rewrite_dasgn_curr :rewrite_dasgn
+
   ##
   # :defn is one of the most complex of all the ASTs in ruby. We do
   # one of 3 different translations:
@@ -91,10 +147,23 @@ module UnifiedRuby
 
     # move block_arg up and in
     block_arg = exp.scope.block.block_arg(true) rescue nil
-    exp.args << block_arg if block_arg
+    if block_arg
+      block = args.block(true)
+      args << :"&#{block_arg.last}"
+      args << block if block
+    end
 
     # patch up attr_accessor methods
-    exp.insert 2, s(:args) if weirdo
+    if weirdo then
+      case
+      when exp.ivar then
+        exp.insert 2, s(:args)
+      when exp.attrset then
+        exp.insert 2, s(:args, :arg)
+      else
+        raise "unknown wierdo: #{wierdo.inpsect}"
+      end
+    end
 
     exp
   end
@@ -119,6 +188,11 @@ module UnifiedRuby
     exp.shift # scope / block / body
   end
 
+  def rewrite_dvar(exp)
+    exp[0] = :lvar
+    exp
+  end
+
   def rewrite_fcall(exp)
     exp[0] = :call
     exp.insert 1, nil
@@ -127,55 +201,105 @@ module UnifiedRuby
     rewrite_call(exp)
   end
 
-  def rewrite_resbody(exp) # TODO: clean up and move to unified
-    result = s()
+  def rewrite_masgn(exp)
+    raise "wtf: #{exp}" unless exp.size == 4
 
-    code = result
-    while exp and exp.first == :resbody do
-      code << exp.shift
-      list = exp.shift || s(:array)
-      body = exp.empty? ? nil : exp.shift
-      exp  = exp.empty? ? nil : exp.shift
+    t, lhs, lhs_splat, rhs = exp
 
-      # code may be nil, :lasgn, or :block
-      case body.first
-      when nil then
-        # do nothing
-      when :lasgn then
-        # TODO: check that it is assigning $!
-        list << body
-        body = nil
-      when :block then
-        # TODO: check that it is assigning $!
-        list << body.delete_at(1) if body[1].first == :lasgn
-      else
-        # do nothing (expression form)
-      end if body
+    lhs ||= s(:array)
+
+    if lhs_splat then
+      lhs_splat = s(:splat, lhs_splat) unless lhs_splat[0] == :splat
+      lhs << lhs_splat
+    end
 
-      code << list << body
-      if exp then
-        code = s()
-        result << code
+    s(t, lhs, rhs).compact
+  end
+
+  def rewrite_op_asgn1(exp)
+    exp[2][0] = :arglist # if exp[2][0] == :array
+    exp
+  end
+
+  def rewrite_rescue(exp)
+    # SKETCHY HACK return exp if exp.size > 4
+    ignored = exp.shift
+    body    = exp.shift unless exp.first.first == :resbody
+    resbody = exp.shift
+    els     = exp.shift unless exp.first.first == :resbody unless exp.empty?
+    rest    = exp.empty? ? nil : exp # graceful re-rewriting (see rewrite_begin)
+
+    resbodies = []
+
+    unless rest then
+      while resbody do
+        resbodies << resbody
+        resbody = resbody.resbody(true)
+      end
+
+      resbodies.each do |resbody|
+        if resbody[2] && resbody[2][0] == :block && resbody[2].size == 2 then
+          resbody[2] = resbody[2][-1]
+        end
       end
+    else
+      resbodies = [resbody] + rest
     end
 
-    if $DEBUG or $TESTING then
-      structure = result.structure
-      raise "result structure wrong: #{structure[0..1].inspect}" unless
-        structure.flatten[0] == :resbody
-      raise "result structure wrong: #{structure[0..1].inspect}" unless
-        s(:array, :splat, :argscat).include? structure.flatten[1]
-      raise "result body wrong: #{structure[2].inspect}" unless
-        structure[2].nil? or not structure[2].empty?
+    resbodies << els if els
+
+    s(:rescue, body, *resbodies).compact
+  end
+
+  def rewrite_resbody(exp)
+    exp[1] ||= s(:array)        # no args
+
+    body = exp[2]
+    if body then
+      case body.first
+      when :lasgn, :iasgn then
+        exp[1] << exp.delete_at(2) if body[-1] == s(:gvar, :$!)
+      when :block then
+        exp[1] << body.delete_at(1) if [:lasgn, :iasgn].include?(body[1][0]) &&
+          body[1][-1] == s(:gvar, :$!)
+      end
     end
 
-    result
+    exp << nil if exp.size == 2 # no body
+
+    exp
+  end
+
+  def rewrite_begin(exp)
+    raise "wtf: #{exp.inspect}" if exp.size > 2
+    exp.last
   end
 
   def rewrite_vcall(exp)
     exp.push nil
     rewrite_fcall(exp)
   end
+
+  def rewrite_yield(exp)
+    # Literal array in yield, not args.
+    if exp.size == 3 and exp.pop == true then
+      # exp[1] = s(:arglist, exp[1])
+    elsif exp.size == 2 && exp.last.first == :array then
+      exp.push(*exp.pop[1..-1])
+    end
+
+    exp
+  end
+
+  def rewrite_super(exp)
+    exp.push(*exp.pop[1..-1]) if exp.size == 2 && exp.last.first == :array
+    exp
+  end
+
+  def rewrite_zarray(exp)
+    exp[0] = :array
+    exp
+  end
 end
 
 ##
@@ -183,5 +307,9 @@ end
 
 class Unifier < SexpProcessor
   include UnifiedRuby
-end
 
+  def initialize
+    super
+    @unsupported.delete :newline
+  end
+end