flay 1.0.0

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2008-11-06
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,7 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/flay
+lib/flay.rb
+test/test_flay.rb

data/README.txt

@@ -0,0 +1,75 @@
+= flay
+
+* http://ruby.sadi.st/
+* http://rubyforge.org/projects/seattlerb
+
+== DESCRIPTION:
+
+Flay analyzes ruby code for structural similarities. Differences in
+literal values, variable, class, method names, whitespace, programming
+style, braces vs do/end, etc are all ignored. Making this totally rad.
+
+== FEATURES/PROBLEMS:
+
+* Differences in literal values, variable, class, and method names are ignored.
+* Differences in whitespace, programming style, braces vs do/end, etc are ignored.
+* Works across files.
+* Reports differences at any level of code.
+
+== TODO:
+
+* Editor integration (emacs, textmate, other contributions welcome).
+* N-way diff reporting... or... something. Not sure.
+* UI improvement suggestions welcome. :)
+
+== SYNOPSIS:
+
+  % flay lib/*.rb
+  Processing unit/itemconfig.rb...
+  
+  Matches found in :when (mass = 572)
+    unit/itemconfig.rb:343
+    unit/itemconfig.rb:379
+    unit/itemconfig.rb:706
+    unit/itemconfig.rb:742
+  
+  Matches found in :when (mass = 500)
+    unit/itemconfig.rb:509
+    unit/itemconfig.rb:539
+    unit/itemconfig.rb:875
+    unit/itemconfig.rb:905
+  ...
+
+== REQUIREMENTS:
+
+* ruby_parser
+* sexp_processor
+
+== INSTALL:
+
+* sudo gem install flay
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Ryan Davis, Seattle.rb
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,19 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe::add_include_dirs("../../sexp_processor/dev/lib",
+                      "../../ruby_parser/dev/lib")
+
+require './lib/flay.rb'
+
+Hoe.new('flay', Flay::VERSION) do |flay|
+  flay.rubyforge_name = 'seattlerb'
+  flay.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
+
+  flay.extra_deps << ['sexp_processor', '>= 3.0.0']
+  flay.extra_deps << ['ruby_parser',    '>= 1.1.0']
+end
+
+# vim: syntax=Ruby

data/bin/flay

@@ -0,0 +1,9 @@
+#!/usr/bin/ruby -s
+
+$m ||= 16
+
+require 'flay'
+
+flay = Flay.new($m.to_i)
+flay.process(*ARGV)
+flay.report

data/lib/flay.rb

@@ -0,0 +1,158 @@
+#!/usr/bin/env ruby -w
+
+$: << "../../sexp_processor/dev/lib" # TODO: remove
+$: << "../../ruby_parser/dev/lib"
+
+require 'rubygems'
+require 'sexp_processor'
+require 'ruby_parser'
+require 'pp' # TODO: remove
+
+class Flay
+  VERSION = '1.0.0'
+
+  attr_reader :hashes
+
+  def initialize(mass = 16)
+    @hashes = Hash.new { |h,k| h[k] = [] }
+    @mass_threshold = mass
+  end
+
+  def process(*files)
+    files.each do |file|
+      warn "Processing #{file}..."
+
+      t = Time.now
+      pt = RubyParser.new.process(File.read(file), file)
+
+      next unless pt # empty files... hahaha, suck.
+
+      t = Time.now
+      pt.deep_each do |node|
+        next unless node.any? { |sub| Sexp === sub }
+        next if node.mass < @mass_threshold
+
+        self.hashes[node.fuzzy_hash] << node
+      end
+    end
+  end
+
+  def prune
+    # prune trees that aren't duped at all, or are too small
+    self.hashes.delete_if { |_,nodes| nodes.size == 1 }
+
+    # extract all subtree hashes from all nodes
+    all_hashes = {}
+    self.hashes.values.each do |nodes|
+      nodes.each do |node|
+        node.all_subhashes.each do |h|
+          all_hashes[h] = true
+        end
+      end
+    end
+
+    # nuke subtrees so we show the biggest matching tree possible
+    self.hashes.delete_if { |h,_| all_hashes[h] }
+  end
+
+  def report prune = nil
+    self.prune
+
+    self.hashes.sort_by { |_,nodes|
+      -(nodes.first.mass * nodes.size)
+    }.each do |_,nodes|
+      next unless nodes.first.first == prune if prune
+      puts
+
+      node = nodes.first
+      puts "Matches found in %p (mass = %d)" %
+        [node.first, nodes.size * node.mass]
+
+      nodes.each do |node|
+        puts "  #{node.file}:#{node.line}"
+      end
+    end
+  end
+end
+
+class Symbol
+  def hash
+    @hash ||= self.to_s.hash
+  end
+end
+
+class Sexp
+  def mass
+    @mass ||= self.structure.flatten.size
+  end
+
+  alias :uncached_structure :structure
+  def structure
+    @structure ||= self.uncached_structure
+  end
+
+  def similarity o
+    l, s, r = self.compare_to o
+    (2.0 * s) / (2.0 * s + l + r)
+  end
+
+  def compare_to they
+    l = s = r = 0
+
+    l_sexp, l_lits = self.partition { |o| Sexp === o }
+    r_sexp, r_lits = they.partition { |o| Sexp === o }
+
+    l += (l_lits - r_lits).size
+    s += (l_lits & r_lits).size
+    r += (r_lits - l_lits).size
+
+    # TODO: I think this is wrong, since it isn't positional. What to do?
+    l_sexp.zip(r_sexp).each do |l_sub, r_sub|
+      l2, s2, r2 = l_sub.compare_to r_sub
+      l += l2
+      s += s2
+      r += r2
+    end
+
+    return l, s, r
+  end
+
+  def fuzzy_hash
+    @fuzzy_hash ||= self.structure.hash
+  end
+
+  def all_subhashes
+    hashes = []
+    self.deep_each do |node|
+      hashes << node.fuzzy_hash
+    end
+    hashes
+  end
+
+  def deep_each(&block)
+    self.each_sexp do |sexp|
+      block[sexp]
+      sexp.deep_each(&block)
+    end
+  end
+
+  def each_sexp
+    self.each do |sexp|
+      next unless Sexp === sexp
+
+      yield sexp
+    end
+  end
+
+  alias :old_inspect :inspect
+  def inspect
+    old_inspect.sub(/\)\Z/, ":h_#{self.fuzzy_hash})")
+  end
+
+  alias :shut_up! :pretty_print
+  def pretty_print(q) # shows the hash TODO: remove
+    q.group(1, 'S(', ')') do
+      q.seplist(self + [":h_#{self.fuzzy_hash}"]) {|v| q.pp v }
+    end
+  end
+end

data/test/test_flay.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/ruby -w
+
+require 'test/unit'
+require 'flay'
+
+class SexpTest < Test::Unit::TestCase
+  def setup
+    # a(1) { |c| d }
+    @s = s(:iter,
+           s(:call, nil, :a, s(:arglist, s(:lit, 1))),
+           s(:lasgn, :c),
+           s(:call, nil, :d, s(:arglist)))
+  end
+
+  def test_mass
+    assert_equal 1, s(:a).mass
+    assert_equal 3, s(:a, s(:b), s(:c)).mass
+    assert_equal 7, @s.mass
+  end
+
+  def test_compare_to
+    s1 = s(:a, :b, :c)
+    s2 = s(:d, :e, :f)
+    assert_equal [3, 0, 3], s1.compare_to(s2), "100% different"
+
+    s1 = s(:a, :b, :c)
+    s2 = s(:a, :b, :c)
+    assert_equal [0, 3, 0], s1.compare_to(s2), "100% same"
+
+    s1 = s(:a, :b, :c, :d)
+    s2 = s(:a, :b, :c, :e)
+    assert_equal [1, 3, 1], s1.compare_to(s2), "1 element different on each"
+
+    s1 = s(:a, :d, :b, :c)
+    s2 = s(:a, :b, :c, :e)
+    assert_equal [1, 3, 1], s1.compare_to(s2), "positionally different"
+
+
+    s1 = s(:a, s(:d), :b, :c)
+    s2 = s(:a, :b, :c, s(:e))
+    assert_equal [1, 3, 1], s1.compare_to(s2), "simple subtree difference"
+  end
+
+  def test_fuzzy_hash
+    s = s(:iter,
+          s(:call, nil, :a, s(:arglist, s(:lit, 1))),
+          s(:lasgn, :c),
+          s(:call, nil, :d, s(:arglist)))
+
+    hash = 955256285
+
+    assert_equal hash, s.fuzzy_hash,             "hand copy"
+    assert_equal hash, @s.fuzzy_hash,            "ivar from setup"
+    assert_equal hash, @s.deep_clone.fuzzy_hash, "deep clone"
+    assert_equal hash, s.deep_clone.fuzzy_hash,  "copy deep clone"
+  end
+
+  def test_all_subhashes
+    expected = [-704571402, -282578980, -35395725,
+                160138040, 815971090, 927228382]
+
+    assert_equal expected, @s.all_subhashes.sort.uniq
+    assert ! @s.all_subhashes.include?(@s.fuzzy_hash)
+
+    x = []
+
+    @s.deep_each do |o|
+      x << o.fuzzy_hash
+    end
+
+    assert_equal expected, x.sort.uniq
+  end
+
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: flay
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Ryan Davis
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-11-06 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sexp_processor
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 3.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: ruby_parser
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.2
+    version: 
+description: Flay analyzes ruby code for structural similarities. Differences in literal values, variable, class, method names, whitespace, programming style, braces vs do/end, etc are all ignored. Making this totally rad.
+email: 
+- ryand-ruby@zenspider.com
+executables: 
+- flay
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- bin/flay
+- lib/flay.rb
+- test/test_flay.rb
+has_rdoc: true
+homepage: http://ruby.sadi.st/
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: seattlerb
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Flay analyzes ruby code for structural similarities
+test_files: 
+- test/test_flay.rb