ParseTree 1.1.1 → 1.2.0

data/History.txt

@@ -1,3 +1,22 @@
+*** 1.2.0 / 2004-11-29
+
++ 9 minor enhancements
+	+ ParseTree now returns symbols for all data except in :str and :lit.
+		+ This WILL break existing code! (as if any exists)
+	+ ParseTree.parse_tree now takes a list of classes, no optional method.
+	+ ParseTree.parse_tree_for_method(klass, method) added.
+	+ ParseTree.parse_tree now returns a list of :class/:module nodes.
+	+ Added -f to parse_tree_show for uber fast demos.
+	+ Added -q to parse_tree_show to NOT use pp so it is mucho faster.
+	+ running with $DEBUG will trace the methods and nodes as processed.
+	+ Added rdoc to everything.
+	+ Gemified
++ 4 bug fixes
+	+ Bus error in iter/for nodes where arg list is empty now fixed.
+	+ Bus error when handling 'def method(*)' is now fixed.
+	+ Unnamed classes and superclasses (ugh) are now handled.
+	+ parse_tree_abc now enumerates actual types rather than every token.
+
 *** 1.1.0 / 2004-11-12
 
 + 2 minor enhancement

data/Makefile

@@ -1,18 +1,22 @@
 RUBY?=ruby
-RUBY_FLAGS?=-w -I.
+RUBY_FLAGS?=-w -Ilib
 RUBY_LIB?=$(shell $(RUBY) -rrbconfig -e 'include Config; print CONFIG["sitelibdir"]')
 PREFIX?=/usr/local
 
-all test:
-	$(RUBY) $(RUBY_FLAGS) test_all.rb
+all test: FORCE
+	$(RUBY) $(RUBY_FLAGS) test/test_all.rb
 
 # we only install test_sexp_processor.rb to help make ruby_to_c's
 # subclass tests work.
 
+docs:
+	rdoc -d -I png --main SexpProcessor -x test_\* -x something.rb
+
 install:
-	cp -f parse_tree.rb test_sexp_processor.rb sexp_processor.rb composite_sexp_processor.rb $(RUBY_LIB)
-	cp -f parse_tree_show $(PREFIX)/bin
-	cp -f parse_tree_abc $(PREFIX)/bin
+	cp -f lib/parse_tree.rb lib/sexp_processor.rb lib/composite_sexp_processor.rb $(RUBY_LIB)
+	cp -f test/test_sexp_processor.rb $(RUBY_LIB)
+	cp -f bin/parse_tree_show $(PREFIX)/bin
+	cp -f bin/parse_tree_abc  $(PREFIX)/bin
 	chmod 444 $(RUBY_LIB)/parse_tree.rb $(RUBY_LIB)/sexp_processor.rb $(RUBY_LIB)/composite_sexp_processor.rb $(RUBY_LIB)/test_sexp_processor.rb
 	chmod 555 $(PREFIX)/bin/parse_tree_show $(PREFIX)/bin/parse_tree_abc
 
@@ -20,6 +24,19 @@ uninstall:
 	rm -f $(RUBY_LIB)/parse_tree.rb $(RUBY_LIB)/sexp_processor.rb $(RUBY_LIB)/composite_sexp_processor.rb $(RUBY_LIB)/test_sexp_processor.rb
 	rm -f $(PREFIX)/bin/parse_tree_show $(PREFIX)/bin/parse_tree_abc
 
+audit:
+	ZenTest composite_sexp_processor.rb sexp_processor.rb test_all.rb test_composite_sexp_processor.rb test_sexp_processor.rb
+
 clean:
-	-rm *~ diff.txt
+	-find . -name \*~ | xargs rm
+	-rm -f diff diff.txt
 	-rm -r $$HOME/.ruby_inline
+	-rm -r doc
+
+demo:
+	echo 1+1 | $(RUBY) $(RUBY_FLAGS) ./bin/parse_tree_show -f
+
+gem:
+	gem ParseTree.gemspec
+
+FORCE:

data/Manifest.txt

@@ -2,13 +2,15 @@ History.txt
 Makefile
 Manifest.txt
 README.txt
-composite_sexp_processor.rb
-parse_tree.rb
-parse_tree_abc
-parse_tree_show
-sexp_processor.rb
-something.rb
-test_all.rb
-test_composite_sexp_processor.rb
-test_parse_tree.rb
-test_sexp_processor.rb
+ParseTree.gemspec
+bin/parse_tree_abc
+bin/parse_tree_deps
+bin/parse_tree_show
+lib/composite_sexp_processor.rb
+lib/parse_tree.rb
+lib/sexp_processor.rb
+test/something.rb
+test/test_all.rb
+test/test_composite_sexp_processor.rb
+test/test_parse_tree.rb
+test/test_sexp_processor.rb

data/ParseTree.gemspec

@@ -0,0 +1,39 @@
+# -*- ruby -*-
+
+require 'rubygems'
+$: << "./lib"
+require 'parse_tree'
+
+spec = Gem::Specification.new do |s|
+
+  s.name = 'ParseTree'
+  s.version = ParseTree::VERSION
+  s.summary = "Extract and enumerate ruby parse trees."
+
+  paragraphs = File.read("README.txt").split(/\n\n+/)
+  s.description = paragraphs[2]
+  puts "Description = #{s.description}"
+
+  s.requirements << "RubyInline."
+  s.files = IO.readlines("Manifest.txt").map {|f| f.chomp }
+
+  s.require_paths = ['lib', 'test']
+  s.autorequire = 'parse_tree'
+
+  s.bindir = "bin"
+  s.executables = s.files.grep(Regexp.new(s.bindir)) { |f| File.basename(f) }
+  puts "Executables = #{s.executables.join(", ")}"
+
+  s.has_rdoc = true
+  s.test_suite_file = "test/test_all.rb"
+
+  s.author = "Ryan Davis"
+  s.email = "ryand-ruby@zenspider.com"
+  s.homepage = "http://www.zenspider.com/ZSS/Products/ParseTree/"
+  s.rubyforge_project = "parsetree"
+end
+
+if $0 == __FILE__
+  Gem.manage_gems
+  Gem::Builder.new(spec).build
+end

data/README.txt

@@ -3,7 +3,7 @@ ParseTree
     support@zenspider.com
 
 ** DESCRIPTION:
-  
+
 ParseTree is a C extension (using RubyInline) that extracts the parse
 tree for an entire class or a specific method and returns it as a
 s-expression (aka sexp) using ruby's arrays, strings, symbols, and

data/bin/parse_tree_abc

@@ -0,0 +1,86 @@
+#!/usr/local/bin/ruby
+
+# ABC metric
+#
+# Assignments, Branches, and Calls
+#
+# A simple way to measure the complexity of a function or method.
+
+begin
+  require 'rubygems'
+  require_gem 'ParseTree'
+rescue LoadError
+  require 'parse_tree'
+end
+
+old_classes = []
+ObjectSpace.each_object(Module) do |klass|
+  old_classes << klass
+end
+
+ARGV.each do |name|
+  require name
+end
+
+new_classes = []
+ObjectSpace.each_object(Module) do |klass|
+  new_classes << klass
+end
+
+score = {}
+
+new_classes -= old_classes
+
+def sexp_types(exp)
+  result = []
+  if Array === exp.first then
+    result = sexp_types(exp.first)
+  else
+    result << exp.shift
+    exp.grep(Array).each do |subexp|
+      result.concat(sexp_types(subexp))
+    end
+  end
+  result
+end
+
+klasses = ParseTree.new.parse_tree(*new_classes)
+klasses.each do |klass|
+  klass.shift # :class
+  klassname = klass.shift
+  klass.shift # superclass
+  methods = klass
+
+  methods.each do |defn|
+    a=b=c=0
+    defn.shift
+    methodname = defn.shift
+    tokens = sexp_types(defn)
+    tokens.each do |token|
+      case token
+      when :attrasgn, :attrset, :dasgn_curr, :iasgn, :lasgn, :masgn then
+        a += 1
+      when :and, :case, :else, :if, :iter, :or, :rescue, :until, :when, :while  then
+        b += 1
+      when :call, :fcall, :super, :vcall, :yield then
+        c += 1
+      when :args, :argscat, :array, :begin, :block, :block_arg, :block_pass, :bool, :cfunc, :colon2, :const, :cvar, :defined, :defn, :dregx, :dstr, :dvar, :dxstr, :ensure, :false, :fbody, :gvar, :hash, :ivar, :lit, :long, :lvar, :match2, :match3, :nil, :not, :nth_ref, :return, :scope, :self, :splat, :str, :to_ary, :true, :unknown, :value, :void, :zarray, :zarray, :zclass, :zsuper then
+        # ignore
+      else
+        puts "unhandled token #{token.inspect}" if $VERBOSE
+      end
+    end
+    key = ["#{klassname}.#{methodname}", a, b, c]
+    val = a+b+c
+    score[key] = val
+  end
+end
+
+puts "Method = assignments + branches + calls = total"
+puts
+count = 1
+score.sort_by { |k,v| v }.reverse.each do |key,val|
+  name, a, b, c = *key
+  printf "%3d) %-50s = %2d + %2d + %2d = %3d\n", count, name, a, b, c, val
+  count += 1
+end

data/bin/parse_tree_deps

@@ -0,0 +1,66 @@
+#!/usr/local/bin/ruby -ws
+
+old_classes = []; new_classes = []
+
+ObjectSpace.each_object(Module) { |klass| old_classes << klass } if defined? $a
+
+require 'pp'
+
+begin
+  require 'rubygems'
+  require_gem 'ParseTree'
+rescue LoadError
+  require 'parse_tree'
+end
+
+require 'sexp_processor'
+
+ObjectSpace.each_object(Module) { |klass| old_classes << klass } unless defined? $a
+
+class DependencyAnalyzer < SexpProcessor
+
+  attr_reader :dependencies
+  attr_accessor :current_class
+
+  def initialize
+    super
+    self.auto_shift_type = true
+    @dependencies = Hash.new { |h,k| h[k] = [] }
+    @current_method = nil
+    @current_class = nil
+  end
+
+  def self.process(*klasses)
+    analyzer = self.new
+    klasses.each do |start_klass|
+      analyzer.current_class = start_klass
+      analyzer.process(ParseTree.new.parse_tree(start_klass))
+    end
+
+    deps = analyzer.dependencies
+    deps.keys.sort.each do |dep_to|
+      dep_from = deps[dep_to]
+      puts "#{dep_to}: #{dep_from.uniq.sort.join(", ")}"
+    end
+  end
+
+  def process_defn(exp)
+    name = exp.shift
+    @current_method = name
+    return s(:defn, name, process(exp.shift), process(exp.shift))
+  end
+
+  def process_const(exp)
+    name = exp.shift
+    const = (defined?($c) ? @current_class.name : "#{@current_class}.#{@current_method}")
+    is_class = ! (Object.const_get(name) rescue nil).nil?
+    @dependencies[name] << const if is_class
+    return s(:const, name)
+  end
+end
+
+if __FILE__ == $0 then
+  ARGV.each { |name| require name }
+  ObjectSpace.each_object(Module) { |klass| new_classes << klass }
+  DependencyAnalyzer.process(*(new_classes - old_classes))
+end

data/bin/parse_tree_show

@@ -0,0 +1,49 @@
+#!/usr/local/bin/ruby -ws
+
+require 'pp'
+
+begin
+  require 'rubygems'
+  require_gem 'ParseTree'
+rescue LoadError
+  require 'parse_tree'
+end
+
+def discover_new_classes_from
+  old_classes = []
+  ObjectSpace.each_object(Module) do |klass|
+    old_classes << klass
+  end
+
+  yield
+
+  new_classes = []
+  ObjectSpace.each_object(Module) do |klass|
+    new_classes << klass
+  end
+
+  new_classes - old_classes
+end
+
+$f = false unless defined? $f
+
+new_classes = discover_new_classes_from do
+  ARGV.unshift "-" if ARGV.empty?
+  ARGV.each do |name|
+    if name == "-" then
+      code = $stdin.read
+      code = "class Example; def example; #{code}; end; end" if $f
+      eval code unless code.nil?
+    else
+      require name
+    end
+  end
+end
+
+result = ParseTree.new.parse_tree(*new_classes)
+unless defined? $q then
+  pp result
+else
+  p result
+end
+

data/lib/composite_sexp_processor.rb

@@ -0,0 +1,43 @@
+require 'sexp_processor'
+
+##
+# Implements the Composite pattern on SexpProcessor. Need we say more?
+#
+# Yeah... probably. Implements a SexpProcessor of SexpProcessors so
+# you can easily chain multiple to each other. At some stage we plan
+# on having all of them run +process+ and but only ever output
+# something when +generate+ is called, allowing for deferred final
+# processing.
+
+class CompositeSexpProcessor < SexpProcessor
+
+  ##
+  # The list o' processors to run.
+
+  attr_reader :processors
+
+  def initialize # :nodoc:
+    super
+    @processors = []
+  end
+
+  ##
+  # Add a +processor+ to the list of processors to run.
+
+  def <<(processor)
+    raise ArgumentError, "Can only add sexp processors" unless
+      SexpProcessor === processor
+    @processors << processor
+  end
+
+  ##
+  # Run +exp+ through all of the processors, returning the final
+  # result.
+
+  def process(exp)
+    @processors.each do |processor|
+      exp = processor.process(exp)
+    end
+    exp
+  end
+end

data/{parse_tree.rb → lib/parse_tree.rb}

@@ -4,12 +4,80 @@ begin
   require 'rubygems'
   require_gem 'RubyInline'
 rescue LoadError
-  require 'inline'
+  require "inline"
 end
 
+##
+# ParseTree is a RubyInline-style extension that accesses and
+# traverses the internal parse tree created by ruby.
+#
+#   class Example
+#     def blah
+#       return 1 + 1
+#     end
+#   end
+#
+#   ParseTree.new.parse_tree(Example)
+#   =>  [[:defn,
+#         "blah",
+#         [:scope,
+#          [:block,
+#           [:args],
+#            [:return, [:call, [:lit, 1], "+", [:array, [:lit, 1]]]]]]]]
+
 class ParseTree
 
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'
+
+  ##
+  # Main driver for ParseTree. Returns an array of arrays containing
+  # the parse tree for +klasses+.
+  #
+  # Structure:
+  #
+  #   [[:class, classname, superclassname, [:defn :method1, ...], ...], ...]
+  #
+  # NOTE: v1.0 - v1.1 had the signature (klass, meth=nil). This wasn't
+  # used much at all and since parse_tree_for_method already existed,
+  # it was deemed more useful to expand this method to do multiple
+  # classes.
+
+  def parse_tree(*klasses)
+    result = []
+    klasses.each do |klass|
+      raise "You should call parse_tree_for_method(#{klasses.first}, #{klass}) instead of parse_tree" if Symbol === klass or String === klass
+      klassname = klass.name
+      klassname = "UnnamedClass_#{klass.object_id}" if klassname.empty?
+      klassname = klassname.to_sym
+
+      code = if Class === klass then
+               superclass = klass.superclass.name
+               superclass = "nil" if superclass.empty?
+               superclass = superclass.to_sym
+               [:class, klassname, superclass]
+             else
+               [:module, klassname]
+             end
+
+      klass.instance_methods(false).sort.each do |m|
+        $stderr.puts "parse_tree_for_method(#{klass}, #{m}):" if $DEBUG
+        code << parse_tree_for_method(klass, m.to_sym)
+      end
+      result << code
+    end
+    return result
+  end
+
+  ##
+  # Returns the parse tree for just one +method+ of a class +klass+.
+  #
+  # Format:
+  #
+  #   [:defn, :name, :body]
+
+  def parse_tree_for_method(klass, method)
+    parse_tree_for_meth(klass, method.to_sym)
+  end
 
   inline do |builder|
     builder.add_type_converter("VALUE", '', '')
@@ -95,6 +163,10 @@ again:
 
   if (node) {
     node_name = ID2SYM(rb_intern(node_type_string[nd_type(node)]));
+    if (RTEST(ruby_debug)) {
+      fputs(node_type_string[nd_type(node)], stderr);
+      fputs("\n", stderr);
+    }
   } else {
     node_name = ID2SYM(rb_intern("ICKY"));
   }
@@ -129,7 +201,7 @@ again_no_block:
 
     case NODE_COLON2:
       add_to_parse_tree(current, node->nd_head);
-      rb_ary_push(current, rb_str_new2(rb_id2name(node->nd_mid)));
+      rb_ary_push(current, ID2SYM(node->nd_mid));
       break;
 
     case NODE_BEGIN:
@@ -199,7 +271,9 @@ again_no_block:
   case NODE_ITER:
   case NODE_FOR:
     add_to_parse_tree(current, node->nd_iter);
-    if (node->nd_var != (NODE *)-1 && node->nd_var != NULL) {
+    if (node->nd_var != (NODE *)1
+        && node->nd_var != (NODE *)2
+        && node->nd_var != NULL) {
       add_to_parse_tree(current, node->nd_var);
     } else {
       rb_ary_push(current, Qnil);
@@ -281,7 +355,7 @@ again_no_block:
   case NODE_VCALL:
     if (nd_type(node) != NODE_FCALL)
       add_to_parse_tree(current, node->nd_recv);
-    rb_ary_push(current, rb_str_new2(rb_id2name(node->nd_mid)));
+    rb_ary_push(current, ID2SYM(node->nd_mid));
     if (node->nd_args || nd_type(node) != NODE_FCALL)
       add_to_parse_tree(current, node->nd_args);
     break;
@@ -337,7 +411,7 @@ again_no_block:
   case NODE_CVASGN:
   case NODE_CVDECL:
   case NODE_GASGN:
-    rb_ary_push(current, rb_str_new2(rb_id2name(node->nd_vid)));
+    rb_ary_push(current, ID2SYM(node->nd_vid));
     add_to_parse_tree(current, node->nd_value);
     break;
 
@@ -398,14 +472,14 @@ again_no_block:
     if (node->nd_defn) {
       if (nd_type(node) == NODE_DEFS)
 	add_to_parse_tree(current, node->nd_recv);
-      rb_ary_push(current, rb_str_new2(rb_id2name(node->nd_mid)));
+      rb_ary_push(current, ID2SYM(node->nd_mid));
       add_to_parse_tree(current, node->nd_defn);
     }
     break;
 
   case NODE_CLASS:
   case NODE_MODULE:
-    rb_ary_push(current, rb_str_new2(rb_id2name((ID)node->nd_cpath->nd_mid)));
+    rb_ary_push(current, ID2SYM((ID)node->nd_cpath->nd_mid));
     if (node->nd_super && nd_type(node) == NODE_CLASS) {
       add_to_parse_tree(current, node->nd_super);
     }
@@ -422,25 +496,36 @@ again_no_block:
 	(node->nd_cnt || node->nd_opt || node->nd_rest != -1)) {
       int i;
       NODE *optnode;
+      long arg_count;
 
       for (i = 0; i < node->nd_cnt; i++) {
         // regular arg names
-        rb_ary_push(current, rb_str_new2(rb_id2name(dump_local_tbl[i + 3])));
+        rb_ary_push(current, ID2SYM(dump_local_tbl[i + 3]));
       }
 
       optnode = node->nd_opt;
       while (optnode) {
         // optional arg names
-        rb_ary_push(current, rb_str_new2(rb_id2name(dump_local_tbl[i + 3])));
+        rb_ary_push(current, ID2SYM(dump_local_tbl[i + 3]));
 	i++;
 	optnode = optnode->nd_next;
       }
-      if (node->nd_rest != -1) {
+
+      arg_count = node->nd_rest;
+      if (arg_count > 0) {
         // *arg name
-        rb_ary_push(current, rb_str_new2(rb_id2name(dump_local_tbl[node->nd_rest + 1])));
+        rb_ary_push(current, ID2SYM(dump_local_tbl[node->nd_rest + 1]));
+      } else if (arg_count == -1) {
+        // nothing to do in this case, handled above
+      } else if (arg_count == -2) {
+        // nothing to do in this case, no name == no use
+      } else {
+        puts("not a clue what this arg value is");
+        exit(1);
       }
+
       optnode = node->nd_opt;
-      // 
+      // block?
       if (optnode) {
 	add_to_parse_tree(current, node->nd_opt);
       }
@@ -454,7 +539,7 @@ again_no_block:
     case NODE_GVAR:
     case NODE_CONST:
     case NODE_ATTRSET:
-      rb_ary_push(current, rb_str_new2(rb_id2name(node->nd_vid)));
+      rb_ary_push(current, ID2SYM(node->nd_vid));
       break;
 
     case NODE_STR:
@@ -506,10 +591,10 @@ again_no_block:
 	goto again_no_block;
     }
   }
-^
+^ # end of add_to_parse_tree block
 
     builder.c %q{
-static VALUE parse_tree_for_method(VALUE klass, VALUE method) {
+static VALUE parse_tree_for_meth(VALUE klass, VALUE method) {
   NODE *node = NULL;
   ID id;
   VALUE result = rb_ary_new();
@@ -519,7 +604,7 @@ static VALUE parse_tree_for_method(VALUE klass, VALUE method) {
   id = rb_to_id(method);
   if (st_lookup(RCLASS(klass)->m_tbl, id, (st_data_t *) &node)) {
     rb_ary_push(result, ID2SYM(rb_intern("defn")));
-    rb_ary_push(result, method);
+    rb_ary_push(result, ID2SYM(id));
     add_to_parse_tree(result, node->nd_body);
   } else {
     rb_ary_push(result, Qnil);
@@ -528,18 +613,5 @@ static VALUE parse_tree_for_method(VALUE klass, VALUE method) {
   return result;
 }
 }
-  end
-
-  def parse_tree(klass, meth=nil)
-    code = []
-    if meth then
-      code = parse_tree_for_method(klass, meth.to_s)
-    else
-      klass.instance_methods(false).sort.each do |m|
-	code << parse_tree_for_method(klass, m)
-      end
-    end
-    return code
-  end
-
-end
+  end # inline call
+end # ParseTree class