ParseTree 1.7.1 → 2.0.0

data/History.txt

@@ -1,3 +1,35 @@
+=== 2.0.0 / 2007-08-01
+
+* 2 major enhancements:
+
+  * Rewrite methods completely rewritten. Rewriters:
+    * are no longer recursive.
+    * are no longer required to empty the sexp coming in.
+    * are depth first, so rewriter gets passed everything already normalized.
+    * keep rewriting until type doesn't change.
+    * are magical goodness.
+  * Added UnifiedRuby module to aid others in writing clean SexpProcessors:
+    * bmethod/dmethod/fbody unified with defn
+    * fcall/vcall unified with call
+    * resbody unified with itself (lots of different forms)
+
+* 5 minor enhancements:
+
+  * Add #modules to Module.
+  * Add Sexp::for shortcut for Sexp.from_array ParseTree.translate(...).
+  * Add parens for :block nodes as appropriate. May be overzealous.
+  * Made add_to_parse_tree global for reuse by other C extensions.
+  * Made test_ruby2ruby MUCH more rigorous with circular testing.
+
+* 6 bug fixes:
+
+  * Added $TESTING = true to pt_testcase.rb. OOPS!
+  * Fixed some circular bugs, mostly by hacking them out, wrt operator precidence.
+  * Fixed splat arg processing when arg has no name.
+  * Fixed trinary operator.
+  * Fixed BMETHOD with no arguments.
+  * Removed hacky "self." thing for defs at top level PT use.
+
 === 1.7.1 / 2007-06-05
 
 * 3 minor enhancements:

data/Manifest.txt

@@ -11,6 +11,7 @@ lib/composite_sexp_processor.rb
 lib/parse_tree.rb
 lib/sexp.rb
 lib/sexp_processor.rb
+lib/unified_ruby.rb
 lib/unique.rb
 test/pt_testcase.rb
 test/something.rb
@@ -19,4 +20,5 @@ test/test_composite_sexp_processor.rb
 test/test_parse_tree.rb
 test/test_sexp.rb
 test/test_sexp_processor.rb
+test/test_unified_ruby.rb
 validate.sh

data/README.txt

@@ -3,12 +3,12 @@ ParseTree
     http://www.zenspider.com/ZSS/Products/ParseTree/
     support@zenspider.com
 
-** DESCRIPTION:
+== DESCRIPTION:
 
 ParseTree is a C extension (using RubyInline) that extracts the parse
 tree for an entire class or a specific method and returns it as a
 s-expression (aka sexp) using ruby's arrays, strings, symbols, and
-integers. 
+integers.
 
 As an example:
 
@@ -32,23 +32,26 @@ becomes:
        nil],
       [:return, [:lit, 0]]]]]
 
-** FEATURES/PROBLEMS:
-  
-+ Uses RubyInline, so it just drops in.
-+ Includes SexpProcessor and CompositeSexpProcessor.
-	+ Allows you to write very clean filters.
-+ Includes parse_tree_show, which lets you quickly snoop code.
-	+ echo "1+1" | parse_tree_show -f for quick snippet output.
-+ Includes parse_tree_abc, which lets you get abc metrics on code.
-	+ abc metrics = numbers of assignments, branches, and calls.
-	+ whitespace independent metric for method complexity.
-+ Includes parse_tree_deps, which shows you basic class level dependencies.
-+ Only works on methods in classes/modules, not arbitrary code.
-+ Does not work on the core classes, as they are not ruby (yet).
+== FEATURES/PROBLEMS:
 
-** SYNOPSYS:
+* Uses RubyInline, so it just drops in.
+* Includes SexpProcessor and CompositeSexpProcessor.
+  * Allows you to write very clean filters.
+* Includes UnifiedRuby, allowing you to automatically rewrite ruby quirks.
+* ParseTree#parse_tree_for_string lets you parse arbitrary strings of ruby.
+* Includes parse_tree_show, which lets you quickly snoop code.
+  * echo "1+1" | parse_tree_show -f for quick snippet output.
+* Includes parse_tree_abc, which lets you get abc metrics on code.
+  * abc metrics = numbers of assignments, branches, and calls.
+  * whitespace independent metric for method complexity.
+* Includes parse_tree_deps, which shows you basic class level dependencies.
+* Does not work on the core classes, as they are not ruby (yet).
 
-  sexp_array = ParseTree.new.parse_tree(klass)
+== SYNOPSYS:
+
+  sexp_array = ParseTree.translate(klass)
+  sexp_array = ParseTree.translate(klass, :method)
+  sexp_array = ParseTree.translate("1+1")
 
 or:
 
@@ -75,20 +78,21 @@ or:
 
   % ./parse_tree_abc myfile.rb
 
-** REQUIREMENTS:
+== REQUIREMENTS:
 
-+ RubyInline 3 or better.
+* RubyInline 3.6 or better.
 
-** INSTALL:
+== INSTALL:
 
-+ sudo rake install
-+ or: sudo gem install ParseTree
+* rake install_gem
+* sudo rake install
+* sudo gem install ParseTree
 
-** LICENSE:
+== LICENSE:
 
 (The MIT License)
 
-Copyright (c) 2001-2004 Ryan Davis, Zen Spider Software
+Copyright (c) 2001-2007 Ryan Davis, Zen Spider Software
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -9,10 +9,11 @@ require './lib/parse_tree.rb'
 Hoe.new("ParseTree", ParseTree::VERSION) do |p|
   p.rubyforge_name = "parsetree"
   p.summary = "Extract and enumerate ruby parse trees."
-  p.description = p.paragraphs_of("README.txt", 2).join("\n\n")
-  p.changes = p.paragraphs_of("History.txt", 1).join("\n\n")
+  p.summary = p.paragraphs_of("README.txt", 2).join("\n\n")
+  p.description = p.paragraphs_of("README.txt", 2..6, 8).join("\n\n")
+  p.changes = p.paragraphs_of("History.txt", 1..6).join("\n\n")
   p.clean_globs << File.expand_path("~/.ruby_inline")
-  p.extra_deps << ['RubyInline', '>= 3.2.0']
+  p.extra_deps << ['RubyInline', '>= 3.6.0']
   p.spec_extras[:require_paths] = proc { |paths| paths << 'test' }
 end
 
@@ -34,8 +35,14 @@ end
 
 desc 'Show what tests are not sorted'
 task :sort do
-  sh "pgrep '^    \\\"(\\w+)' test/pt_testcase.rb | cut -f 2 -d\\\" > x"
-  sh "sort x > y"
-  sh "diff x y"
-  sh "rm -f x y"
+  begin
+    sh "pgrep '^    \\\"(\\w+)' test/pt_testcase.rb | cut -f 2 -d\\\" > x"
+    sh "sort x > y"
+    sh "diff x y"
+
+    sh 'for f in lib/*.rb; do echo $f; grep "^ *def " $f | grep -v sort=skip > x; sort x > y; echo $f; echo; diff x y; done'
+    sh 'for f in test/test_*.rb; do echo $f; grep "^ *def.test_" $f > x; sort x > y; echo $f; echo; diff x y; done'
+  ensure
+    sh 'rm -f x y'
+  end
 end

data/lib/parse_tree.rb

@@ -7,6 +7,19 @@ begin require 'rubygems'; rescue LoadError; end
 
 require 'inline'
 
+class Module
+  def modules
+    ancestors[1..-1]
+  end
+end
+
+class Class
+  def modules
+    a = self.ancestors
+    a[1..a.index(superclass)-1]
+  end
+end
+
 ##
 # ParseTree is a RubyInline-style extension that accesses and
 # traverses the internal parse tree created by ruby.
@@ -28,7 +41,7 @@ require 'inline'
 
 class ParseTree
 
-  VERSION = '1.7.1'
+  VERSION = '2.0.0'
 
   ##
   # Front end translation method.
@@ -109,7 +122,17 @@ class ParseTree
       # protected methods are included in instance_methods, go figure!
 
       method_names.sort.each do |m|
-        code << parse_tree_for_method(klass, m.to_sym)
+        r = parse_tree_for_method(klass, m.to_sym)
+p m => r if r == [nil]
+        code << r
+      end
+
+      klass.modules.each do |mod| # TODO: add a test for this damnit
+        mod.instance_methods.each do |m|
+          r = parse_tree_for_method(mod, m.to_sym)
+p m => r if r == [nil]
+          code << r
+        end
       end
 
       klass.singleton_methods(false).sort.each do |m|
@@ -131,7 +154,6 @@ class ParseTree
   def parse_tree_for_method(klass, method, is_cls_meth=false)
     $stderr.puts "** parse_tree_for_method(#{klass}, #{method}):" if $DEBUG
     r = parse_tree_for_meth(klass, method.to_sym, @include_newlines, is_cls_meth)
-    r[1] = :"self.#{r[1]}" if is_cls_meth
     r
   end
 
@@ -306,11 +328,11 @@ class ParseTree
   ##
   # add_to_parse_tree(ary, node, include_newlines, local_variables)
 
-  builder.c_raw %Q@
-static void add_to_parse_tree(VALUE ary,
-                              NODE * n,
-                              VALUE newlines,
-                              ID * locals) {
+  builder.prefix %Q@
+void add_to_parse_tree(VALUE ary,
+                       NODE * n,
+                       VALUE newlines,
+                       ID * locals) {
   NODE * volatile node = n;
   NODE * volatile contnode = NULL;
   VALUE old_ary = Qnil;
@@ -544,7 +566,11 @@ again_no_block:
     {
       struct BLOCK *data;
       Data_Get_Struct(node->nd_cval, struct BLOCK, data);
-      add_to_parse_tree(current, data->var, newlines, locals);
+      if (data->var == 0 || data->var == (NODE *)1 || data->var == (NODE *)2) {
+        rb_ary_push(current, Qnil);
+      } else {
+        add_to_parse_tree(current, data->var, newlines, locals);
+      }
       add_to_parse_tree(current, data->body, newlines, locals);
       break;
     }
@@ -773,7 +799,11 @@ again_no_block:
 
       if (arg_count > 0) {
         // *arg name
-        VALUE sym = rb_str_intern(rb_str_plus(rb_str_new2("*"), rb_str_new2(rb_id2name(locals[i + 3]))));
+        VALUE sym = rb_str_new2("*");
+        if (locals[i + 3]) {
+          rb_str_concat(sym, rb_str_new2(rb_id2name(locals[i + 3])));
+        }
+        sym = rb_str_intern(sym);
         rb_ary_push(current, sym);
       } else if (arg_count == 0) {
         // nothing to do in this case, empty list
@@ -878,8 +908,9 @@ again_no_block:
     break;
 
   case NODE_CFUNC:
-    rb_ary_push(current, INT2FIX(node->nd_cfnc));
-    rb_ary_push(current, INT2FIX(node->nd_argc));
+  case NODE_IFUNC:
+    rb_ary_push(current, INT2NUM((long)node->nd_cfnc));
+    rb_ary_push(current, INT2NUM(node->nd_argc));
     break;
 
 #{if_version :<, "1.9", "#if 0"}
@@ -895,7 +926,6 @@ again_no_block:
   // I think these are all runtime only... not positive but...
   case NODE_MEMO:               // enum.c zip
   case NODE_CREF:
-  case NODE_IFUNC:
   // #defines:
   // case NODE_LMASK:
   // case NODE_LSHIFT:
@@ -942,7 +972,10 @@ static VALUE parse_tree_for_meth(VALUE klass, VALUE method, VALUE newlines, VALU
   }
   if (st_lookup(RCLASS(klass)->m_tbl, id, &n)) {
     node = (NODE*)n;
-    rb_ary_push(result, ID2SYM(rb_intern("defn")));
+    rb_ary_push(result, ID2SYM(rb_intern(is_cls_meth ? "defs": "defn")));
+    if (is_cls_meth) {
+      rb_ary_push(result, rb_ary_new3(1, ID2SYM(rb_intern("self"))));
+    }
     rb_ary_push(result, ID2SYM(id));
     add_to_parse_tree(result, node->nd_body, newlines, NULL);
   } else {

data/lib/sexp.rb

@@ -1,11 +1,13 @@
+require 'parse_tree'
+
+$TESTING ||= false # unless defined $TESTING
+
 ##
 # Sexps are the basic storage mechanism of SexpProcessor.  Sexps have
 # a +type+ (to be renamed +node_type+) which is the first element of
 # the Sexp. The type is used by SexpProcessor to determine whom to
 # dispatch the Sexp to for processing.
 
-$TESTING ||= false # unless defined $TESTING
-
 class Sexp < Array # ZenTest FULL
 
   @@array_types = [ :array, :args, ]
@@ -17,6 +19,28 @@ class Sexp < Array # ZenTest FULL
     super(args)
   end
 
+  ##
+  # Creates a new Sexp for +klass+ or +method+ in +klass+.
+  #
+  # If +walk_ancestors+ is true and +method+ is provided, walks the ancestors
+  # of +klass+ until a method definition is found.
+
+  def self.for(klass, method = nil, walk_ancestors = false)
+    sexp = if walk_ancestors and method then
+             klass.ancestors.each do |klass|
+               sexp = ParseTree.translate klass, method
+               break sexp unless sexp == [nil]
+             end
+           else
+             ParseTree.translate klass, method
+           end
+
+    Sexp.from_array sexp
+  end
+
+  ##
+  # Creates a new Sexp from Array +a+, typically from ParseTree::translate.
+
   def self.from_array(a)
     ary = Array === a ? a : [a]
 
@@ -44,6 +68,9 @@ class Sexp < Array # ZenTest FULL
     end
   end
 
+  ##
+  # Returns true if this Sexp's pattern matches +sexp+.
+
   def ===(sexp)
     return nil unless Sexp === sexp
     pattern = self # this is just for my brain
@@ -57,6 +84,9 @@ class Sexp < Array # ZenTest FULL
     return nil
   end
 
+  ##
+  # Returns true if this Sexp matches +pattern+.  (Opposite of #===.)
+
   def =~(pattern)
     return pattern === self
   end
@@ -97,6 +127,9 @@ class Sexp < Array # ZenTest FULL
     end
   end
 
+  ##
+  # Replaces all Sexps matching +pattern+ with Sexp +repl+.
+
   def gsub(pattern, repl)
     return repl if pattern == self
 
@@ -117,6 +150,9 @@ class Sexp < Array # ZenTest FULL
     return "s(#{sexp_str})"
   end
 
+  ##
+  # Returns the node named +node+, deleting it if +delete+ is true.
+
   def method_missing(meth, delete=false)
     matches = find_all { | sexp | Sexp === sexp and sexp.first == meth }
 
@@ -155,7 +191,7 @@ class Sexp < Array # ZenTest FULL
   end if $DEBUG or $TESTING
 
   ##
-  # Returnes the bare bones structure of the sexp.
+  # Returns the bare bones structure of the sexp.
   # s(:a, :b, s(:c, :d), :e) => s(:a, s(:c))
 
   def structure
@@ -171,6 +207,9 @@ class Sexp < Array # ZenTest FULL
     result
   end
 
+  ##
+  # Replaces the Sexp matching +pattern+ with +repl+.
+
   def sub(pattern, repl)
     return repl.dup if pattern == self
 
@@ -213,6 +252,10 @@ end
 class SexpMatchSpecial < Sexp; end
 
 class SexpAny < SexpMatchSpecial
+  def ==(o)
+    Sexp === o
+  end
+
   def ===(o)
     return Sexp === o
   end

data/lib/sexp_processor.rb

@@ -164,25 +164,16 @@ class SexpProcessor
   end
 
   def rewrite(exp)
-    meth = @rewriters[exp.first]
-    if meth then
-      r = self.send(meth, exp)
-      self.assert_empty(meth, exp, nil) if @require_empty
-      r
-    else
-      result = exp.class.new
-      until exp.empty? do
-        sub_exp = exp.shift
-        sub_result = nil
-
-        if Array === sub_exp then
-          result << rewrite(sub_exp)
-        else
-          result << sub_exp
-        end
-      end
-      result
-    end
+    exp.map! { |sub| Array === sub ? rewrite(sub) : sub }
+
+    type = exp.first
+    begin
+      meth = @rewriters[type]
+      exp  = self.send(meth, exp) if meth
+      old_type, type = type, exp.first
+    end until old_type == type
+
+    exp
   end
 
   ##
@@ -221,6 +212,11 @@ class SexpProcessor
 
     exp = self.rewrite(exp) if @process_level == 1
 
+    if @debug.has_key? type then
+      str = exp.inspect
+      puts "// DEBUG (rewritten): #{str}" if str =~ @debug[type]
+    end
+
     # now do a pass with the real processor (or generic)
     meth = @processors[type] || @default_method
     if meth then