danmayer-metric_fu 2.1.2

data/lib/base/graph.rb

@@ -0,0 +1,44 @@
+module MetricFu
+
+  def self.graph
+    @graph ||= Graph.new
+  end
+
+  class Graph
+
+    attr_accessor :clazz
+
+    def initialize
+      self.clazz = []
+    end
+
+    def add(graph_type, graph_engine)
+      grapher_name = graph_type.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase } + graph_engine.to_s.capitalize + "Grapher"
+      self.clazz.push MetricFu.const_get(grapher_name).new
+    end
+
+
+    def generate
+      return if self.clazz.empty?
+      puts "Generating graphs"
+      Dir[File.join(MetricFu.data_directory, '*.yml')].sort.each do |metric_file|
+        puts "Generating graphs for #{metric_file}"
+        date_parts = year_month_day_from_filename(metric_file)
+        metrics = YAML::load(File.open(metric_file))
+
+        self.clazz.each do |grapher|
+          grapher.get_metrics(metrics, "#{date_parts[:m]}/#{date_parts[:d]}")
+        end
+      end
+      self.clazz.each do |grapher|
+        grapher.graph!
+      end
+    end
+
+    private
+    def year_month_day_from_filename(path_to_file_with_date)
+      date = path_to_file_with_date.match(/\/(\d+).yml$/)[1]
+      {:y => date[0..3].to_i, :m => date[4..5].to_i, :d => date[6..7].to_i}
+    end
+  end
+end

data/lib/base/line_numbers.rb

@@ -0,0 +1,74 @@
+require 'ruby_parser'
+module MetricFu
+  class LineNumbers
+
+    def initialize(contents)
+      rp = RubyParser.new
+      @locations = {}
+      file_sexp = rp.parse(contents)
+      case file_sexp[0]
+      when :class
+        process_class(file_sexp)
+      when :module
+        process_module(file_sexp)
+      when :block
+        file_sexp.each_of_type(:class) { |sexp| process_class(sexp) }
+      else
+      end
+    rescue Exception
+      #catch errors for files ruby_parser fails on
+      @locations
+    end
+
+    def in_method? line_number
+      !!@locations.detect do |method_name, line_number_range|
+        line_number_range.include?(line_number)
+      end
+    end
+
+    def method_at_line line_number
+      found_method_and_range = @locations.detect do |method_name, line_number_range|
+        line_number_range.include?(line_number)
+      end
+      return nil unless found_method_and_range
+      found_method_and_range.first
+    end
+
+    def start_line_for_method(method)
+      return nil unless @locations.has_key?(method)
+      @locations[method].first
+    end
+
+    private
+
+    def process_module(sexp)
+      module_name = sexp[1]
+      sexp.each_of_type(:class) do |sexp|
+        process_class(sexp, module_name)
+        hide_methods_from_next_round(sexp)
+      end
+      process_class(sexp)
+    end
+
+    def process_class(sexp, module_name=nil)
+      class_name = sexp[1]
+      process_class_self_blocks(sexp, class_name)
+      module_name_string = module_name ? "#{module_name}::" : nil
+      sexp.each_of_type(:defn) { |s| @locations["#{module_name_string}#{class_name}##{s[1]}"] = (s.line)..(s.last.line) }
+      sexp.each_of_type(:defs) { |s| @locations["#{module_name_string}#{class_name}::#{s[2]}"] = (s.line)..(s.last.line) }
+    end
+
+    def process_class_self_blocks(sexp, class_name)
+      sexp.each_of_type(:sclass) do |sexp_in_class_self_block|
+        sexp_in_class_self_block.each_of_type(:defn) { |s| @locations["#{class_name}::#{s[1]}"] = (s.line)..(s.last.line) }
+        hide_methods_from_next_round(sexp_in_class_self_block)
+      end
+    end
+
+    def hide_methods_from_next_round(sexp)
+      sexp.find_and_replace_all(:defn, :ignore_me)
+      sexp.find_and_replace_all(:defs, :ignore_me)
+    end
+
+  end
+end

data/lib/base/location.rb

@@ -0,0 +1,85 @@
+module MetricFu
+  class Location
+    include Comparable
+
+    attr_accessor :class_name, :method_name, :file_path, :simple_method_name, :hash
+
+    def self.get(file_path, class_name, method_name)
+      # This could be more 'confident' using Maybe, but we want it to be as fast as possible
+      file_path_copy = file_path == nil ? nil : file_path.clone
+      class_name_copy = class_name == nil ? nil : class_name.clone
+      method_name_copy = method_name == nil ? nil : method_name.clone
+      key = [file_path_copy, class_name_copy, method_name_copy]
+      @@locations ||= {}
+      if @@locations.has_key?(key)
+        @@locations[key]
+      else
+        location = self.new(file_path_copy, class_name_copy, method_name_copy)
+        @@locations[key] = location
+        location.freeze  # we cache a lot of method call results, so we want location to be immutable
+        location
+      end
+    end
+
+    def initialize(file_path, class_name, method_name)
+      @file_path = file_path
+      @class_name = class_name
+      @method_name = method_name
+      @simple_method_name = @method_name.sub(@class_name,'') unless @method_name == nil
+      @hash = [@file_path, @class_name, @method_name].hash
+    end
+
+    # TODO - we need this method (and hash accessor above) as a temporary hack where we're using Location as a hash key
+    def eql?(other)
+      [self.file_path.to_s, self.class_name.to_s, self.method_name.to_s] == [other.file_path.to_s, other.class_name.to_s, other.method_name.to_s]
+    end
+    # END we need these methods as a temporary hack where we're using Location as a hash key
+
+    def self.for(class_or_method_name)
+      class_or_method_name = strip_modules(class_or_method_name)
+      if(class_or_method_name)
+        begin
+          match = class_or_method_name.match(/(.*)((\.|\#|\:\:[a-z])(.+))/)
+        rescue => error
+          #new error during port to metric_fu occasionally a unintialized
+          #MatchData object shows up here. Not expected.
+          match = nil
+        end
+
+        # reek reports the method with :: not # on modules like
+        # module ApplicationHelper \n def signed_in?, convert it so it records correctly
+        # but classes have to start with a capital letter... HACK for REEK bug, reported underlying issue to REEK
+        if(match)
+          class_name = strip_modules(match[1])
+          method_name = class_or_method_name.gsub(/\:\:/,"#")
+        else
+          class_name = strip_modules(class_or_method_name)
+          method_name = nil
+        end
+      else
+        class_name = nil
+        method_name = nil
+      end
+      self.get(nil, class_name, method_name)
+    end
+
+    def <=>(other)
+      [self.file_path.to_s, self.class_name.to_s, self.method_name.to_s] <=> [other.file_path.to_s, other.class_name.to_s, other.method_name.to_s]
+    end
+
+    private
+
+    def self.strip_modules(class_or_method_name)
+      # reek reports the method with :: not # on modules like 
+      # module ApplicationHelper \n def signed_in?, convert it so it records correctly
+      # but classes have to start with a capital letter... HACK for REEK bug, reported underlying issue to REEK
+      if(class_or_method_name=~/\:\:[A-Z]/)
+        class_or_method_name.split("::").last
+      else
+        class_or_method_name
+      end
+
+    end
+
+  end
+end

data/lib/base/md5_tracker.rb

@@ -0,0 +1,52 @@
+require 'digest/md5'
+require 'fileutils'
+
+module MetricFu
+  class MD5Tracker
+
+    @@unchanged_md5s = []
+
+    class << self
+      def md5_dir(path_to_file, base_dir)
+        File.join(base_dir,
+                  path_to_file.split('/')[0..-2].join('/'))
+      end
+
+      def md5_file(path_to_file, base_dir)
+        File.join(md5_dir(path_to_file, base_dir),
+                  path_to_file.split('/').last.sub(/\.[a-z]+/, '.md5'))
+      end
+
+      def track(path_to_file, base_dir)
+        md5 = Digest::MD5.hexdigest(File.read(path_to_file))
+        FileUtils.mkdir_p(md5_dir(path_to_file, base_dir), :verbose => false)
+        f = File.new(md5_file(path_to_file, base_dir), "w")
+        f.puts(md5)
+        f.close
+        md5
+      end
+
+      def file_changed?(path_to_file, base_dir)
+        orig_md5_file = md5_file(path_to_file, base_dir)
+        return !!track(path_to_file, base_dir) unless File.exist?(orig_md5_file)
+
+        current_md5 = ""
+        file = File.open(orig_md5_file, 'r')
+        file.each_line { |line| current_md5 << line }
+        file.close
+        current_md5.chomp!
+
+        new_md5 = Digest::MD5.hexdigest(File.read(path_to_file))
+        new_md5.chomp!
+
+        @@unchanged_md5s << path_to_file if new_md5 == current_md5
+
+        return new_md5 != current_md5
+      end
+
+      def file_already_counted?(path_to_file)
+        return @@unchanged_md5s.include?(path_to_file)
+      end
+    end
+  end
+end

data/lib/base/metric_analyzer.rb

@@ -0,0 +1,404 @@
+class AnalysisError < RuntimeError; end;
+
+class MetricAnalyzer
+
+  COMMON_COLUMNS = %w{metric}
+  GRANULARITIES =  %w{file_path class_name method_name}
+
+  attr_accessor :table
+
+  def initialize(yaml)
+    if(yaml.is_a?(String))
+       @yaml = YAML.load(yaml)
+     else
+       @yaml = yaml
+     end
+    @file_ranking = MetricFu::Ranking.new
+    @class_ranking = MetricFu::Ranking.new
+    @method_ranking = MetricFu::Ranking.new
+    rankings = [@file_ranking, @class_ranking, @method_ranking]
+
+    tool_analyzers = [ReekAnalyzer.new, RoodiAnalyzer.new,
+    FlogAnalyzer.new, ChurnAnalyzer.new, SaikuroAnalyzer.new,
+    FlayAnalyzer.new, StatsAnalyzer.new, RcovAnalyzer.new]
+    # TODO There is likely a clash that will happen between
+    # column names eventually. We should probably auto-prefix
+    # them (e.g. "roodi_problem")
+    columns = COMMON_COLUMNS + GRANULARITIES + tool_analyzers.map{|analyzer| analyzer.columns}.flatten
+
+    @table = make_table(columns)
+
+    # These tables are an optimization. They contain subsets of the master table.
+    # TODO - these should be pushed into the Table class now
+    @tool_tables   = make_table_hash(columns)
+    @file_tables   = make_table_hash(columns)
+    @class_tables  = make_table_hash(columns)
+    @method_tables = make_table_hash(columns)
+
+    tool_analyzers.each do |analyzer|
+      analyzer.generate_records(@yaml[analyzer.name], @table)
+    end
+
+    build_lookups!(table)
+    process_rows!(table)
+
+    tool_analyzers.each do |analyzer|
+      GRANULARITIES.each do |granularity|
+        metric_ranking = calculate_metric_scores(granularity, analyzer)
+        add_to_master_ranking(ranking(granularity), metric_ranking, analyzer)
+      end
+    end
+
+    rankings.each do |ranking|
+      ranking.delete(nil)
+    end
+  end
+
+  def location(item, value)
+    sub_table = get_sub_table(item, value)
+    if(sub_table.length==0)
+      raise AnalysisError, "The #{item.to_s} '#{value.to_s}' does not have any rows in the analysis table"
+    else
+      first_row = sub_table[0]
+      case item
+      when :class
+        MetricFu::Location.get(first_row.file_path, first_row.class_name, nil)
+      when :method
+        MetricFu::Location.get(first_row.file_path, first_row.class_name, first_row.method_name)
+      when :file
+        MetricFu::Location.get(first_row.file_path, nil, nil)
+      else
+        raise ArgumentError, "Item must be :class, :method, or :file"
+      end
+    end
+  end
+
+  #todo redo as item,value, options = {}
+  # Note that the other option for 'details' is :detailed (this isn't
+  # at all clear from this method itself
+  def problems_with(item, value, details = :summary, exclude_details = [])
+    sub_table = get_sub_table(item, value)
+    #grouping = Ruport::Data::Grouping.new(sub_table, :by => 'metric')
+    grouping = get_grouping(sub_table, :by => 'metric')
+    problems = {}
+    grouping.each do |metric, table|
+      if details == :summary || exclude_details.include?(metric)
+        problems[metric] = present_group(metric,table)
+      else
+        problems[metric] = present_group_details(metric,table)
+      end
+    end
+    problems
+  end
+
+  def worst_methods(size = nil)
+    @method_ranking.top(size)
+  end
+
+  def worst_classes(size = nil)
+    @class_ranking.top(size)
+  end
+
+  def worst_files(size = nil)
+    @file_ranking.top(size)
+  end
+
+  private
+
+  def get_grouping(table, opts)
+    #Ruport::Data::Grouping.new(table, opts)
+    Grouping.new(table, opts)
+    #@grouping_cache ||= {}
+    #@grouping_cache.fetch(grouping_key(table,opts)) do
+    #  @grouping_cache[grouping_key(table,opts)] = Ruport::Data::Grouping.new(table, opts)
+    #end
+  end
+
+  def grouping_key(table, opts)
+    "table #{table.object_id} opts #{opts.inspect}"
+  end
+
+  def build_lookups!(table)
+    @class_and_method_to_file ||= {}
+    # Build a mapping from [class,method] => filename
+    # (and make sure the mapping is unique)
+    table.each do |row|
+      # We know that Saikuro provides the wrong data
+      next if row['metric'] == :saikuro
+      key = [row['class_name'], row['method_name']]
+      file_path = row['file_path']
+      @class_and_method_to_file[key] ||= file_path
+    end
+  end
+
+  def process_rows!(table)
+    # Correct incorrect rows in the table
+    table.each do |row|
+      row_metric = row['metric'] #perf optimization
+      if row_metric == :saikuro
+        fix_row_file_path!(row)
+      end
+      @tool_tables[row_metric] << row
+      @file_tables[row["file_path"]] << row
+      @class_tables[row["class_name"]] << row
+      @method_tables[row["method_name"]] << row
+    end
+  end
+
+  def fix_row_file_path!(row)
+    # We know that Saikuro rows are broken
+    # next unless row['metric'] == :saikuro
+    key = [row['class_name'], row['method_name']]
+    current_file_path = row['file_path'].to_s
+    correct_file_path = @class_and_method_to_file[key]
+    if(correct_file_path!=nil && correct_file_path.include?(current_file_path))
+      row['file_path'] = correct_file_path
+    else
+      # There wasn't an exact match, so we can do a substring match
+      matching_file_path = file_paths.detect {|file_path|
+        file_path!=nil && file_path.include?(current_file_path)
+      }
+      if(matching_file_path)
+        row['file_path'] = matching_file_path
+      end
+    end
+  end
+
+  def file_paths
+    @file_paths ||= @table.column('file_path').uniq
+  end
+
+  def ranking(column_name)
+    case column_name
+    when "file_path"
+      @file_ranking
+    when "class_name"
+      @class_ranking
+    when "method_name"
+      @method_ranking
+    else
+      raise ArgumentError, "Invalid column name #{column_name}"
+    end
+  end
+
+  def calculate_metric_scores(granularity, analyzer)
+    metric_ranking = MetricFu::Ranking.new
+    metric_violations = @tool_tables[analyzer.name]
+    metric_violations.each do |row|
+      location = row[granularity]
+      metric_ranking[location] ||= []
+      metric_ranking[location] << analyzer.map(row)
+    end
+
+    metric_ranking.each do |item, scores|
+      metric_ranking[item] = analyzer.reduce(scores)
+    end
+
+    metric_ranking
+  end
+
+  def add_to_master_ranking(master_ranking, metric_ranking, analyzer)
+    metric_ranking.each do |item, _|
+      master_ranking[item] ||= 0
+      master_ranking[item] += analyzer.score(metric_ranking, item) # scaling? Do we just add in the raw score?
+    end
+  end
+
+  def most_common_column(column_name, size)
+    #grouping = Ruport::Data::Grouping.new(@table,
+    #                                      :by => column_name,
+    #                                      :order => lambda { |g| -g.size})
+    get_grouping(@table, :by => column_name, :order => lambda {|g| -g.size})
+    values = []
+    grouping.each do |value, _|
+      values << value if value!=nil
+      if(values.size==size)
+        break
+      end
+    end
+    return nil if values.empty?
+    if(values.size == 1)
+      return values.first
+    else
+      return values
+    end
+  end
+
+  # TODO: As we get fancier, the presenter should
+  # be its own class, not just a method with a long
+  # case statement
+  def present_group(metric, group)
+    occurences = group.size
+    case(metric)
+    when :reek
+      "found #{occurences} code smells"
+    when :roodi
+      "found #{occurences} design problems"
+    when :churn
+      "detected high level of churn (changed #{group[0].times_changed} times)"
+    when :flog
+      complexity = get_mean(group.column("score"))
+      "#{"average " if occurences > 1}complexity is %.1f" % complexity
+    when :saikuro
+      complexity = get_mean(group.column("complexity"))
+      "#{"average " if occurences > 1}complexity is %.1f" % complexity
+    when :flay
+      "found #{occurences} code duplications"
+    when :rcov
+      average_code_uncoverage = get_mean(group.column("percentage_uncovered"))
+      "#{"average " if occurences > 1}uncovered code is %.1f%" % average_code_uncoverage
+    else
+      raise AnalysisError, "Unknown metric #{metric}"
+    end
+  end
+
+  def present_group_details(metric, group)
+    occurences = group.size
+    case(metric)
+    when :reek
+      message = "found #{occurences} code smells<br/>"
+      group.each do |item|
+        type    = item.data["reek__type_name"]
+        reek_message = item.data["reek__message"]
+        message << "* #{type}: #{reek_message}<br/>"
+      end
+      message
+    when :roodi
+      message = "found #{occurences} design problems<br/>"
+      group.each do |item|
+        problem    = item.data["problems"]
+        message << "* #{problem}<br/>"
+      end
+      message
+    when :churn
+      "detected high level of churn (changed #{group[0].times_changed} times)"
+    when :flog
+      complexity = get_mean(group.column("score"))
+      "#{"average " if occurences > 1}complexity is %.1f" % complexity
+    when :saikuro
+      complexity = get_mean(group.column("complexity"))
+      "#{"average " if occurences > 1}complexity is %.1f" % complexity
+    when :flay
+      message = "found #{occurences} code duplications<br/>"
+      group.each do |item|
+        problem    = item.data["flay_reason"]
+        problem    = problem.gsub(/^[0-9]*\)/,'')
+        problem    = problem.gsub(/files\:/,' <br>&nbsp;&nbsp;&nbsp;files:')
+        message << "* #{problem}<br/>"
+      end
+      message
+    else
+      raise AnalysisError, "Unknown metric #{metric}"
+    end
+  end
+
+  def make_table_hash(columns)
+    Hash.new { |hash, key|
+      hash[key] = make_table(columns)
+    }
+  end
+
+  def make_table(columns)
+    Table.new(:column_names => columns)
+  end
+
+  def get_sub_table(item, value)
+    tables = {
+      :class  => @class_tables,
+      :method => @method_tables,
+      :file   => @file_tables,
+      :tool   => @tool_tables
+    }.fetch(item) do
+      raise ArgumentError, "Item must be :class, :method, or :file"
+    end
+    tables[value]
+  end
+
+  def get_mean(collection)
+    collection_length = collection.length
+    sum = 0
+    sum = collection.inject( nil ) { |sum,x| sum ? sum+x : x }
+    (sum.to_f / collection_length.to_f)
+  end
+
+end
+
+class Record
+
+  attr_reader :data
+
+  def initialize(data, columns)
+    @data = data
+    @columns = columns
+  end
+
+  def method_missing(name, *args, &block)
+    key = name.to_s
+    if @data.has_key?(key)
+      @data[key]
+    elsif @columns.member?(key)
+      nil
+    else
+      super(name, *args, &block)
+    end
+  end
+
+  def []=(key, value)
+     @data[key]=value
+  end
+
+  def [](key)
+    @data[key]
+  end
+
+  def keys
+    @data.keys
+  end
+
+  def has_key?(key)
+    @data.has_key?(key)
+  end
+
+  def attributes
+    @columns
+  end
+
+end
+
+class Grouping
+
+  def initialize(table, opts)
+    column_name = opts.fetch(:by)
+    order = opts.fetch(:order) { nil }
+    hash = {}
+    if column_name.to_sym == :metric # special optimized case
+      hash = table.group_by_metric
+    else
+      table.each do |row|
+        hash[row[column_name]] ||= Table.new(:column_names => row.attributes)
+        hash[row[column_name]] << row
+      end
+    end
+    if order
+      @arr = hash.sort_by &order
+    else
+      @arr = hash.to_a
+    end
+  end
+
+  def [](key)
+    @arr.each do |group_key, table|
+      return table if group_key == key
+    end
+    return nil
+  end
+
+  def each
+    @arr.each do |value, rows|
+      yield value, rows
+    end
+  end
+
+end
+
+