covet 0.1.0

data/lib/covet/cli.rb

@@ -0,0 +1,254 @@
+require 'optparse'
+require_relative 'log_file'
+require_relative 'collection_filter'
+require_relative 'version'
+
+module Covet
+  class CLI
+    class << self
+      attr_accessor :options
+    end
+
+    def initialize(argv)
+      @argv = argv
+    end
+
+    # TODO: process cmdline options for
+    #   - specify VCS [ ]
+    #   - specify test seed (ordering)
+    #   - stats (show filtered files, files to run vs files to ignore, etc.)
+    def run
+      options = nil
+      begin
+        options = Options.parse!(@argv)
+        self.class.options = options
+      rescue OptionParser::InvalidArgument, OptionParser::InvalidOption => e
+        Kernel.abort "Error: #{e.message}"
+      end
+
+      if options[:collect_cmdline] && !options[:collect_cmdline].empty?
+        cmd = options[:collect_cmdline]
+        env_options = { 'COVET_COLLECT' => '1' }
+        if options[:collect_gem_whitelist].any?
+          env_options['COVET_GEM_WHITELIST'] = %Q("#{options[:collect_gem_whitelist].join(',')}")
+        end
+        if options[:test_runner] != Options::DEFAULTS[:test_runner]
+          env_options['COVET_TEST_RUNNER'] = %Q("#{options[:test_runner]}")
+        end
+        env_list_str = env_options.to_a.map { |ary| ary[0] + '=' + ary[1] }.join(' ')
+        cmd = %Q(#{env_list_str} #{cmd})
+        puts cmd
+        puts "Collecting coverage information for each test method..."
+        Kernel.exec cmd
+      end
+
+      revision = options[:revision]
+      line_changes = nil # establish scope
+      begin
+        line_changes = LineChangesVCS.changes_since(revision)
+      rescue Rugged::RepositoryError
+        Kernel.abort "Error: #{Dir.pwd} is not a git repository. " \
+          "Make sure you're in the project root."
+      rescue Rugged::Error, Rugged::InvalidError, TypeError
+        Kernel.abort "Error: #{options[:revision]} is not a valid revision reference in #{options[:VCS]}"
+      end
+      if line_changes.empty?
+        if revision.to_s == 'last_commit'
+          revision = "last commit" # prettier output below
+        end
+        puts "# No changes since #{revision}. You can specify the #{options[:VCS]} revision using the --revision option."
+        Kernel.exit
+      end
+
+      cov_map = Hash.new { |h, file| h[file] = Hash.new { |i, line| i[line] = [] } }
+      logfile = LogFile.new(:mode => 'r')
+
+      if logfile.file_exists?
+
+        run_stats = {}
+        # Read in the coverage info
+        logfile.load_each_buf! do |buf|
+          buf.each do |args|
+            if args[0] == 'base' # first value logged
+              run_options = args.last
+              if run_options['version'] != Covet::VERSION
+                warn "Warning - the run log was created with another version of covet " \
+                "(#{run_options['version']}), which is not guaranteed to be compatible " \
+                "with this version of covet (#{Covet::VERSION}). Please run 'covet -c' again."
+              end
+              next
+            end
+
+            if args[0] == 'stats' # last value logged
+              run_stats.update(args.last)
+              next
+            end
+
+            desc = args[0]
+            delta = args[1]
+            next if delta.nil? # no coverage difference
+            #stats = args[2]
+
+            delta.each_pair do |fname, lines_hash|
+              file_map = cov_map[fname]
+              lines_hash.each do |line, _executions|
+                # add the test name to the map. Multiple tests can execute the same
+                # line, so we need to use an array.
+                file_map[line.to_i] << desc
+              end
+            end
+          end
+        end
+
+        git_repo = VCS::Git.find_git_repo_path!
+
+        to_run = []
+        line_changes.each do |(file, line)|
+          full_path = File.join(git_repo, file)
+          relative_to_pwd = file
+          if git_repo != Dir.pwd
+            relative_to_pwd = full_path.sub(Dir.pwd, '').sub(File::SEPARATOR, '')
+          end
+          # NOTE: here, `file` is a filename starting from the GIT path (not necessarily `Dir.pwd`)
+          # if the actual test files changed, then we need to run the whole file again.
+          if relative_to_pwd.start_with?(*Covet.test_directories)
+            if relative_to_pwd.start_with?("test#{File::SEPARATOR}") && relative_to_pwd.end_with?('_test.rb', '_spec.rb')
+              to_run << [file, full_path] unless to_run.include?([file, full_path])
+              # We have to disable the method filter in this case because we
+              # don't know the method names of all these methods in this file.
+              # TODO: save this information in the coverage log file and use it here.
+              options[:disable_test_method_filter] = true
+            elsif relative_to_pwd.start_with?("spec#{File::SEPARATOR}") && relative_to_pwd.end_with?('_test.rb', '_spec.rb')
+              to_run << [file, full_path] unless to_run.include?([file, full_path])
+              # We have to disable the method filter in this case because we
+              # don't know the method names of all these methods in this file.
+              # TODO: save this information in the coverage log file and use it here.
+              options[:disable_test_method_filter] = true
+            end
+            next
+          end
+          # library code changes
+          cov_map[full_path][line].each do |desc|
+            to_run << [file, desc] unless to_run.include?([file, desc])
+          end
+        end
+        if ENV['COVET_INVERT_RUN_LIST'] == '1' # NOTE: for debugging covet only
+          to_run_fnames = to_run.map { |(_file, desc)| desc.split('#').first }.flatten.uniq
+          all_fnames = Dir.glob("{#{Covet.test_directories.join(',')}}/**/*_{test,spec}.rb").to_a.map { |fname| File.expand_path(fname, Dir.pwd) }
+          to_run = (all_fnames - to_run_fnames).map { |fname| [fname, "#{fname}##{fname}"] }.sort_by do |ary|
+            ary[1].split('#').first
+          end
+        end
+        if options[:exec_run_list]
+          if to_run.empty?
+            puts "# No test cases to run"
+          else
+            cmdline = Covet.cmdline_for_run_list(to_run)
+            puts cmdline
+            Kernel.exec cmdline
+          end
+        elsif options[:print_run_list]
+          if to_run.empty?
+            puts "# No test cases to run"
+          else
+            if options[:print_run_list_format] == :"test-runner"
+              puts Covet.cmdline_for_run_list(to_run)
+            else
+              # TODO: show not just the files but also the methods in each file
+              puts "You need to run:"
+              to_run.uniq! { |(_file, desc)| desc.split('#').first }
+              to_run.each do |(_file, desc)|
+                puts " - #{desc.split('#').first}"
+              end
+            end
+          end
+        end
+      else
+        Kernel.abort "Error: The coverage log file doesn't exist.\n" \
+          "You need to collect info first with 'covet -c $TEST_CMD'\n" \
+          "Ex: covet -c \"rake test\""
+      end
+    end
+  end
+
+  module Options
+    DEFAULTS = {
+      :collect_cmdline => nil,
+      :VCS => :git,
+      :revision => :last_commit,
+      :test_order => :random_seeded, # one of [:random_seeded, :random, or :ordered]
+      :test_runner => :minitest, # ENV['COVET_TEST_RUNNER']
+      :exec_run_list => false,
+      :disable_test_method_filter => false,
+      :print_run_list => true,
+      :print_run_list_format => :simple,
+      :collect_gem_whitelist => [], # ENV['COVET_GEM_WHITELIST']
+      :debug => false, # TODO: use
+      :verbose => 0, # TODO: levels 0, 1, 2, maybe?
+    }.freeze
+
+    # @return Hash
+    def self.parse!(argv)
+      options = DEFAULTS.dup
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: covet [options]"
+        opts.separator ""
+        opts.separator "Specific options:"
+
+        opts.on('-c', '--collect CMDLINE', 'collect coverage information for test run of given cmdline') do |cmdline|
+          options[:collect_cmdline] = cmdline
+        end
+        opts.on('--whitelist-gems GEMS', Array, 'whitelist given gems for collection phase') do |gems|
+          options[:collect_gem_whitelist] = gems
+          gems.each { |gem| CollectionFilter.whitelist_gem(gem) }
+        end
+        opts.on('-f', '--print-fmt FMT', "Format run list - 'simple' (default) or 'test-runner'") do |fmt|
+          case fmt
+          # TODO: add 'json' format to dump run list in JSON
+          when 'simple', 'test-runner'
+            options[:print_run_list_format] = fmt.intern
+          else
+            raise OptionParser::InvalidArgument,
+              "Valid values: 'simple', 'test-runner'"
+          end
+        end
+        opts.on('-e', '--exec', 'Execute run list') do
+          options[:print_run_list] = false
+          options[:exec_run_list] = true
+        end
+        opts.on('--disable-method-filter', 'When executing run list, run all test methods for each file') do
+          options[:disable_test_method_filter] = true
+        end
+        opts.on('-r', '--revision REVISION', 'VCS Revision (defaults to last commit)') do |rev|
+          options[:revision] = rev
+        end
+        opts.on('-t', '--test-runner RUNNER') do |runner|
+          begin
+            Covet.test_runner = runner
+          rescue ArgumentError => e
+            Kernel.abort "Error: #{e.message}"
+          end
+          options[:test_runner] = runner.intern
+        end
+        #opts.on('-o', '--test-order ORDER', 'Specify test order for collection phase.') do |order|
+          #begin
+            #Covet.test_order = order.to_s
+          #rescue ArgumentError => e
+            #Kernel.abort "Error: #{e.message}"
+          #end
+        #end
+        opts.on_tail('-v', '--version', 'Show covet version') do
+          puts VERSION
+          Kernel.exit
+        end
+        opts.on('-h', '--help', 'Show this message') do
+          puts opts
+          Kernel.exit
+        end
+      end.parse!(argv)
+
+      options
+    end
+  end
+end

data/lib/covet/collection_compressor.rb

@@ -0,0 +1,20 @@
+module Covet
+  module CollectionCompressor
+
+    # Turn sparse Array returned from `Coverage.peek_result` into
+    # more compact representation - a Hash of only the lines that
+    # were executed at least once.
+    # @param [Hash] coverage_info
+    def self.compress(coverage_info)
+      ret = {}
+      coverage_info.each do |fname, cov_ary|
+        ret[fname] ||= {}
+        cov_ary.each_with_index do |times_run, idx|
+          next if times_run.to_i == 0
+          ret[fname][idx+1] = times_run # lineno = idx + 1
+        end
+      end
+      ret
+    end
+  end
+end

data/lib/covet/collection_filter.rb

@@ -0,0 +1,113 @@
+require 'rbconfig'
+require 'set'
+
+module Covet
+  # Responsible for filtering out files that shouldn't be logged in the
+  # `run_log` during the coverage collection phase. For instance, if using
+  # the `minitest` test runner, we shouldn't log coverage information
+  # for internal `minitest` methods (unless we're using covet ON `minitest` itself),
+  # The same goes for `rake`, etc. This minimizes the amount of JSON we have to save
+  # in the `run_log`, and also the amount of processing we have to do on the `run_log`
+  # structure when generating the `run_list`.
+  module CollectionFilter
+    if (gem_whitelist = ENV['COVET_GEM_WHITELIST'])
+      @@gem_whitelist = gem_whitelist.split(',').reject(&:empty?)
+    else
+      @@gem_whitelist = []
+    end
+    @@custom_filters = [] # @var Array<Proc>
+    @@file_whitelist = [] # @var Array<String>, full file path whitelist
+    @@regexp_whitelist = [] # @var Array<Regexp>
+    @@filter_stats = Hash.new { |h,k| h[k] = Set.new } # @var Hash<String => Set>, holds info on which files were blacklisted or whitelisted from `run_log`
+
+    # @param String|Symbol gem_name
+    def self.whitelist_gem(gem_name)
+      @@gem_whitelist << gem_name.to_s
+    end
+
+    # FIXME: should take filename AND method name
+    # @param Proc filter, arity = 1, takes filename
+    def self.add_custom_filter(&filter)
+      @@custom_filters << filter
+    end
+    def self.add_to_file_whitelist(fname)
+      unless fname.start_with?(File::SEPARATOR)
+        raise ArgumentError, "expected #{fname} to be an absolute path"
+      end
+      @@file_whitelist << fname
+    end
+    def self.add_to_regexp_whitelist(regexp)
+      @@regexp_whitelist << regexp
+    end
+
+    def self.files_filtered
+      Hash[@@filter_stats.map { |k, v| [k, v.to_a] }]
+    end
+
+    # @return Hash
+    def self.filter(raw_coverage_info)
+      raw_coverage_info = raw_coverage_info.dup
+      # NOTE: The list of activated gems isn't cached, because it could be
+      # that a test method activates a gem or calls code that activates a
+      # gem. In that case, we want to omit the newly activated gem from the
+      # run log as well (unless it's whitelisted).
+      gem_base_dirs_to_omit = Gem.loaded_specs.values.reject do |spec|
+        @@gem_whitelist.include?(spec.name)
+      end.map do |spec|
+        spec.full_gem_path
+      end
+
+      files_to_omit = []
+
+      # find file names to omit from the run log
+      raw_coverage_info.each do |filename, _|
+        if whitelisted_filename?(filename)
+          @@filter_stats['whitelisted'] << filename
+          next # don't omit
+        end
+
+        if filename.start_with?(ruby_stdlib_dir)
+          @@filter_stats['stdlib_files'] << filename
+          files_to_omit << filename
+          next
+        end
+
+        omitted = gem_base_dirs_to_omit.find do |gem_base_dir|
+          if filename.start_with?(gem_base_dir)
+            @@filter_stats['gem_files'] << filename
+            files_to_omit << filename
+          end
+        end
+        next if omitted
+
+        # custom filters
+        @@custom_filters.find do |filter|
+          if filter.call(filename)
+            @@filter_stats['custom_filters'] << filename
+            files_to_omit << filename
+          end
+        end
+      end
+
+      files_to_omit.each do |fname|
+        raw_coverage_info.delete(fname)
+      end
+
+      raw_coverage_info
+    end
+
+    private
+
+      def self.whitelisted_filename?(filename)
+        if @@file_whitelist.include?(filename)
+          return true
+        end
+        @@regexp_whitelist.find { |re| re === filename }
+      end
+
+      def self.ruby_stdlib_dir
+        RbConfig::CONFIG['libdir']
+      end
+
+  end
+end

data/lib/covet/line_changes_vcs.rb

@@ -0,0 +1,13 @@
+module Covet
+  # Gets file and line numbers that have changed since an arbitrary point in the VCS's
+  # (Version Control System's) history.
+  module LineChangesVCS
+    # @return Set<Array>
+    def self.changes_since(since = :last_commit)
+      require_relative "vcs/#{Covet.vcs}"
+      Covet::VCS.const_get(Covet.vcs.capitalize).changes_since(since)
+    rescue LoadError
+      raise NotImplementedError, "#{self.class} can't get line changes using VCS: #{Covet.vcs} (not implemented)"
+    end
+  end
+end

data/lib/covet/log_collection.rb

@@ -0,0 +1,51 @@
+require_relative 'log_file'
+
+module Covet
+  class LogCollection
+    attr_reader :flushes, :size
+
+    def initialize(options = {})
+      @bufsize = options[:bufsize] || 100 # max log buffer size to keep in memory
+      @log_file = LogFile.new(:filename => options[:filename], :mode => 'w')
+      @buf = []
+      @flushes = 0
+      @size = 0
+    end
+
+    # @param [Array] logs
+    def <<(logs)
+      unless Array === logs
+        raise TypeError, "expecting Array, got #{logs.class}"
+      end
+      @buf << logs
+      if @buf.size == @bufsize
+        flush!
+      end
+      @size += 1
+      true
+    end
+    alias :append :<<
+
+    def finish!
+      if @flushes == 0 && @buf.size == 0
+        return # avoid writing to file if no collections
+      end
+      flush! if @buf.any?
+      @log_file.write_end
+      true
+    end
+
+    private
+
+      # Flushes buffer to file
+      def flush!
+        if @flushes == 0
+          @log_file.write_start
+        end
+        @log_file.write_buf(@buf)
+        @buf.clear
+        @flushes += 1
+      end
+
+  end
+end