flay 1.2.1 → 1.3.0

data/History.txt

@@ -1,3 +1,13 @@
+=== 1.3.0 / 2009-06-23
+
+* 5 minor enhancements:
+
+  * Added --summary to display flay scores per file.
+  * Added --verbose to display processing progress.
+  * Protect against syntax errors in bad code and continue flaying.
+  * Removed fuzzy matching. Never got it to feel right. Slow. Broken on 1.9
+  * Renamed --verbose to --diff.
+
 === 1.2.1 / 2009-03-16
 
 * 3 minor enhancements:

data/Rakefile

@@ -6,47 +6,16 @@ require 'hoe'
 Hoe::add_include_dirs("../../sexp_processor/dev/lib",
                       "../../ruby_parser/dev/lib")
 
-begin
-  require 'flay'
-rescue LoadError
-  load 'lib/flay.rb'
-end
-
-Hoe.new('flay', Flay::VERSION) do |flay|
-  flay.rubyforge_name = 'seattlerb'
-  flay.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
-
-  flay.flay_threshold = 250
-
-  flay.extra_deps << ['sexp_processor', '>= 3.0.0']
-  flay.extra_deps << ['ruby_parser',    '>= 1.1.0']
-end
-
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |t|
-    pattern = ENV['PATTERN'] || 'test/test_*.rb'
+Hoe.plugin :seattlerb
 
-    t.test_files = FileList[pattern]
-    t.verbose = true
-    t.rcov_opts << "--threshold 80"
-    t.rcov_opts << "--no-color"
-  end
+Hoe.spec 'flay' do
+  developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
 
-  task :rcov_info do
-    pattern = ENV['PATTERN'] || "test/test_*.rb"
-    ruby "-Ilib -S rcov --text-report --save coverage.info -x rcov,sexp_processor --test-unit-only #{pattern}"
-  end
+  self.rubyforge_name = 'seattlerb'
+  self.flay_threshold = 250
 
-  task :rcov_overlay do
-    rcov, eol = Marshal.load(File.read("coverage.info")).last[ENV["FILE"]], 1
-    puts rcov[:lines].zip(rcov[:coverage]).map { |line, coverage|
-      bol, eol = eol, eol + line.length
-      [bol, eol, "#ffcccc"] unless coverage
-    }.compact.inspect
-  end
-rescue LoadError
-  # skip
+  extra_deps << ['sexp_processor', '~> 3.0']
+  extra_deps << ['ruby_parser',    '~> 2.0']
 end
 
-# vim: syntax=Ruby
+# vim: syntax=ruby

data/lib/flay.rb

@@ -8,16 +8,15 @@ require 'rubygems'
 require 'sexp_processor'
 require 'ruby_parser'
 
-abort "update rubygems to >= 1.3.1" unless  Gem.respond_to? :find_files
-
 class Flay
-  VERSION = '1.2.1'
+  VERSION = '1.3.0'
 
   def self.default_options
     {
-      :fuzzy   => false,
-      :verbose => false,
+      :diff    => false,
       :mass    => 16,
+      :summary => false,
+      :verbose => false,
     }
   end
 
@@ -37,23 +36,37 @@ class Flay
         exit
       end
 
-      opts.on('-f', '--fuzzy', "Attempt to do fuzzy similarities. (SLOW)") do
-        options[:fuzzy] = true
+      opts.on('-f', '--fuzzy', "DEAD: fuzzy similarities.") do
+        abort "--fuzzy is no longer supported. Sorry. It sucked."
       end
 
       opts.on('-m', '--mass MASS', Integer, "Sets mass threshold") do |m|
         options[:mass] = m.to_i
       end
 
-      opts.on('-v', '--verbose', "Verbose. Display N-Way diff for ruby.") do
+      opts.on('-v', '--verbose', "Verbose. Show progress processing files.") do
         options[:verbose] = true
       end
 
+      opts.on('-d', '--diff', "Diff Mode. Display N-Way diff for ruby.") do
+        options[:diff] = true
+      end
+
+      opts.on('-s', '--summary', "Summarize. Show flay score per file only.") do
+        options[:summary] = true
+      end
+
       extensions = ['rb'] + Flay.load_plugins
 
       opts.separator ""
       opts.separator "Known extensions: #{extensions.join(', ')}"
-    end.parse!
+
+      begin
+        opts.parse!
+      rescue => e
+        abort "#{e}\n\n#{opts}"
+      end
+    end
 
     options
   end
@@ -85,6 +98,8 @@ class Flay
       @@plugins = plugins.map { |f| File.basename(f, '.rb').sub(/^flay_/, '') }
     end
     @@plugins
+  rescue
+    # ignore
   end
 
   attr_accessor :mass_threshold, :total, :identical, :masses
@@ -99,12 +114,12 @@ class Flay
     self.total          = 0
     self.mass_threshold = @option[:mass]
 
-    require 'ruby2ruby' if @option[:verbose]
+    require 'ruby2ruby' if @option[:diff]
   end
 
   def process(*files) # TODO: rename from process - should act as SexpProcessor
     files.each do |file|
-      warn "Processing #{file}"
+      warn "Processing #{file}" if option[:verbose]
 
       ext = File.extname(file).sub(/^\./, '')
       ext = "rb" if ext.nil? || ext.empty?
@@ -115,21 +130,23 @@ class Flay
         msg = "process_rb"
       end
 
-      sexp = begin
-               send msg, file
-             rescue => e
-               warn "  #{e.message.strip}"
-               warn "  skipping #{file}"
-               nil
-             end
+      begin
+        sexp = begin
+                 send msg, file
+               rescue => e
+                 warn "  #{e.message.strip}"
+                 warn "  skipping #{file}"
+                 nil
+               end
 
-      next unless sexp
+        next unless sexp
 
-      process_sexp sexp
+        process_sexp sexp
+      rescue SyntaxError => e
+        warn "  skipping #{file}: #{e.message}"
+      end
     end
 
-    process_fuzzy_similarities if option[:fuzzy]
-
     analyze
   end
 
@@ -157,42 +174,6 @@ class Flay
     end
   end
 
-  def process_fuzzy_similarities
-    all_hashes, detected = {}, {}
-
-    self.hashes.values.each do |nodes|
-      nodes.each do |node|
-        next if node.mass > 4 * self.mass_threshold
-        # TODO: try out with fuzzy_hash
-        # all_hashes[node] = node.grep(Sexp).map { |s| [s.hash] * s.mass }.flatten
-        all_hashes[node] = node.grep(Sexp).map { |s| [s.hash] }.flatten
-      end
-    end
-
-    # warn "looking for copy/paste/edit code across #{all_hashes.size} nodes"
-
-    all_hashes = all_hashes.to_a
-    all_hashes.each_with_index do |(s1, h1), i|
-      similar = [s1]
-      all_hashes[i+1..-1].each do |(s2, h2)|
-        next if detected[h2]
-        intersection = h1.intersection h2
-        max = [h1.size, h2.size].max
-        if intersection.size >= max * 0.60 then
-          similarity = s1.similarity(s2)
-          if similarity > 0.60 then
-            similar << s2
-            detected[h2] = true
-          else
-            p [similarity, s1, s2]
-          end
-        end
-      end
-
-      self.hashes[similar.first.hash].push(*similar) if similar.size > 1
-    end
-  end
-
   def prune
     # prune trees that aren't duped at all, or are too small
     self.hashes.delete_if { |_,nodes| nodes.size == 1 }
@@ -242,10 +223,33 @@ class Flay
     groups.flatten.join("\n")
   end
 
+  def summary
+    score = Hash.new 0
+
+    masses.each do |hash, mass|
+      sexps = hashes[hash]
+      mass_per_file = mass.to_f / sexps.size
+      sexps.each do |sexp|
+        score[sexp.file] += mass_per_file
+      end
+    end
+
+    score
+  end
+
   def report prune = nil
     puts "Total score (lower is better) = #{self.total}"
     puts
 
+    if option[:summary] then
+
+      self.summary.sort_by { |_,v| -v }.each do |file, score|
+        puts "%8.2f: %s" % [score, file]
+      end
+
+      return
+    end
+
     count = 0
     masses.sort_by { |h,m| [-m, hashes[h].first.file] }.each do |hash, mass|
       nodes = hashes[hash]
@@ -265,16 +269,16 @@ class Flay
       puts "%d) %s code found in %p (mass%s = %d)" %
         [count, match, node.first, bonus, mass]
 
-      nodes.each_with_index do |node, i|
-        if option[:verbose] then
+      nodes.each_with_index do |x, i|
+        if option[:diff] then
           c = (?A + i).chr
-          puts "  #{c}: #{node.file}:#{node.line}"
+          puts "  #{c}: #{x.file}:#{x.line}"
         else
-          puts "  #{node.file}:#{node.line}"
+          puts "  #{x.file}:#{x.line}"
         end
       end
 
-      if option[:verbose] then
+      if option[:diff] then
         puts
         r2r = Ruby2Ruby.new
         puts n_way_diff(*nodes.map { |s| r2r.process(s.deep_clone) })

data/test/test_flay.rb

@@ -5,11 +5,15 @@ require 'flay'
 
 require 'pp' # TODO: remove
 
+ON_1_9 = RUBY_VERSION =~ /1\.9/
+SKIP_1_9 = true && ON_1_9 # HACK
+
 class Symbol # for testing only, makes the tests concrete
   def hash
     to_s.hash
   end
 
+  alias :crap :<=> if :blah.respond_to? :<=>
   def <=> o
     Symbol === o && self.to_s <=> o.to_s
   end
@@ -65,11 +69,11 @@ class TestSexp < Test::Unit::TestCase
     assert_equal hash, @s.fuzzy_hash,            "ivar from setup"
     assert_equal hash, @s.deep_clone.fuzzy_hash, "deep clone"
     assert_equal hash, s.deep_clone.fuzzy_hash,  "copy deep clone"
-  end
+  end unless SKIP_1_9
 
   def test_all_subhashes
     expected = [-704571402, -282578980, -35395725,
-                160138040, 815971090, 927228382] # , 955256285]
+                160138040, 815971090, 927228382]
 
     assert_equal expected, @s.all_subhashes.sort.uniq
 
@@ -80,7 +84,7 @@ class TestSexp < Test::Unit::TestCase
     end
 
     assert_equal expected, x.sort.uniq
-  end
+  end unless SKIP_1_9
 
   def test_process_sexp
     flay = Flay.new
@@ -123,7 +127,6 @@ class TestSexp < Test::Unit::TestCase
                 [:block],
                 [:call, :call],
                 [:call],
-                # HACK [:defn],
                 [:if],
                 [:return],
                 [:return],
@@ -133,7 +136,7 @@ class TestSexp < Test::Unit::TestCase
 
     actual = flay.hashes.values.map { |sexps| sexps.map { |sexp| sexp.first } }
 
-    assert_equal expected, actual.sort_by { |a| a.first.to_s }
+    assert_equal expected, actual.sort_by { |a| a.inspect }
   end
 
   def test_process_sexp_no_structure
@@ -142,68 +145,6 @@ class TestSexp < Test::Unit::TestCase
 
     assert flay.hashes.empty?
   end
-
-  def test_process_fuzzy_similarities
-    flay = Flay.new :mass => 7
-
-    s1 = RubyParser.new.process("def w(n); a; b; c; d; e; end")
-    s2 = RubyParser.new.process("def x(n); a;    c;    e; end")
-
-    flay.process_sexp s1
-    flay.process_sexp s2
-
-    flay.process_fuzzy_similarities
-
-    b1 = s1.scope.block
-    b2 = s2.scope.block
-
-    assert_equal [b2, b1], flay.hashes[b2.hash]
-  end
-
-  def test_process_fuzzy_similarities_2
-    flay = Flay.new :mass => 7
-
-    s1 = RubyParser.new.process("def w(n); a; b; c; d; e; end")
-    s2 = RubyParser.new.process("def x(n); a;    c;    e; end")
-    s3 = RubyParser.new.process("def y(n); a; f; c; g; e; end")
-
-    flay.process_sexp s1
-    flay.process_sexp s2
-    flay.process_sexp s3
-
-    flay.process_fuzzy_similarities
-
-    b1 = s1.scope.block
-    b2 = s2.scope.block
-    b3 = s3.scope.block
-
-    assert_equal [b3, b2, b1], flay.hashes[b3.hash]
-  end
-
-  def test_process_fuzzy_similarities_3
-    flay = Flay.new :mass => 7
-
-    s1 = RubyParser.new.process("def w (n); a; b;      c; d;      e; end")
-    s2 = RubyParser.new.process("def x (n); a;         c;         e; end")
-    s3 = RubyParser.new.process("def y (n); a; f;      c; g;      e; end")
-    s4 = RubyParser.new.process("def z (n); f; g;      h; i;      j; end")
-    s5 = RubyParser.new.process("def w1(n); a; b if x; c; d if y; e; end")
-
-    flay.process_sexp s1
-    flay.process_sexp s2
-    flay.process_sexp s3
-    flay.process_sexp s4
-    flay.process_sexp s5
-
-    flay.process_fuzzy_similarities
-
-    b1 = s1.scope.block
-    b2 = s2.scope.block
-    b3 = s3.scope.block
-    b5 = s5.scope.block
-
-    assert_equal [b3, b5, b2, b1], flay.hashes[b3.hash]
-  end
 end
 
 class ArrayIntersectionTests < Test::Unit::TestCase

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: flay
 version: !ruby/object:Gem::Version 
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors: 
 - Ryan Davis
@@ -30,7 +30,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2009-03-16 00:00:00 -07:00
+date: 2009-06-23 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -39,9 +39,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: 3.0.0
+        version: "3.0"
     version: 
 - !ruby/object:Gem::Dependency 
   name: ruby_parser
@@ -49,9 +49,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: 1.1.0
+        version: "2.0"
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -61,9 +61,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.11.0
+        version: 2.3.0
     version: 
-description: Flay analyzes code for structural similarities. Differences in literal values, variable, class, method names, whitespace, programming style, braces vs do/end, etc are all ignored. Making this totally rad.
+description: |-
+  Flay analyzes code for structural similarities. Differences in literal
+  values, variable, class, method names, whitespace, programming style,
+  braces vs do/end, etc are all ignored. Making this totally rad.
 email: 
 - ryand-ruby@zenspider.com
 executables: 
@@ -87,6 +90,8 @@ files:
 - test/test_flay.rb
 has_rdoc: true
 homepage: http://ruby.sadi.st/
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --main
@@ -108,9 +113,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: seattlerb
-rubygems_version: 1.3.1
+rubygems_version: 1.3.4
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: Flay analyzes code for structural similarities
 test_files: 
 - test/test_flay.rb