coverband 4.2.0 → 4.2.2

data/lib/coverband/collectors/delta.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
+
 module Coverband
   module Collectors
     class Delta
-      @semaphore = Mutex.new
-      @@previous_coverage = nil
+      @@previous_coverage = {}
+      @@stubs = {}
+
       attr_reader :current_coverage
 
       def initialize(current_coverage)
@@ -12,54 +14,58 @@ module Coverband
 
       class RubyCoverage
         def self.results
-          ::Coverage.peek_result.dup
+          if Coverband.configuration.use_oneshot_lines_coverage
+            ::Coverage.result(clear: true, stop: false)
+          else
+            ::Coverage.peek_result
+          end
         end
       end
 
       def self.results(process_coverage = RubyCoverage)
-        @semaphore.synchronize do
-          set_default_results
-          new(process_coverage.results).results
-        end
-      end
-
-      def self.previous_results
-        @@previous_coverage
-      end
-
-      def self.set_default_results
-        @@previous_coverage ||= {}
+        coverage_results = process_coverage.results
+        new(coverage_results).results
       end
 
       def results
-        new_results = generate
-        @@previous_coverage = current_coverage
-        new_results
+        if Coverband.configuration.use_oneshot_lines_coverage
+          transform_oneshot_lines_results(current_coverage)
+        else
+          new_results = generate
+          @@previous_coverage = current_coverage unless Coverband.configuration.simulate_oneshot_lines_coverage
+          new_results
+        end
       end
 
       def self.reset
-        @@previous_coverage = nil
+        @@previous_coverage = {}
       end
 
       private
 
       def generate
         current_coverage.each_with_object({}) do |(file, line_counts), new_results|
-          if @@previous_coverage[file]
-            new_results[file] = array_diff(line_counts, @@previous_coverage[file])
-          else
-            new_results[file] = line_counts
-          end
+          new_results[file] = if @@previous_coverage && @@previous_coverage[file]
+                                array_diff(line_counts, @@previous_coverage[file])
+                              else
+                                line_counts
+                              end
         end
       end
 
       def array_diff(latest, original)
         latest.map.with_index do |v, i|
-          if (v && original[i])
-            [0, v - original[i]].max
-          else
-            nil
+          [0, v - original[i]].max if v && original[i]
+        end
+      end
+
+      def transform_oneshot_lines_results(results)
+        results.each_with_object({}) do |(file, coverage), new_results|
+          @@stubs[file] ||= ::Coverage.line_stub(file)
+          transformed_line_counts = coverage[:oneshot_lines].each_with_object(@@stubs[file].dup) do |line_number, line_counts|
+            line_counts[line_number - 1] = 1
           end
+          new_results[file] = transformed_line_counts
         end
       end
     end

data/lib/coverband/collectors/view_tracker.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'singleton'
+
+module Coverband
+  module Collectors
+    ###
+    # This class tracks view file usage via ActiveSupport::Notifications
+    #
+    # This is a port of Flatfoot, a project I open sourced years ago, but am now rolling into Coverband
+    # https://github.com/livingsocial/flatfoot
+    #
+    # TODO: test and ensure slim, haml, and other support
+    ###
+    class ViewTracker
+      DEFAULT_TARGET = Dir.glob('app/views/**/*.html.erb').reject { |file| file.match(/(_mailer)/) }
+      attr_accessor :target, :logged_views, :views_to_record
+      attr_reader :logger, :roots, :store, :ignore_patterns
+
+      def initialize(options = {})
+        raise NotImplementedError, 'View Tracker requires Rails 4 or greater' unless self.class.supported_version?
+
+        @project_directory = File.expand_path(Coverband.configuration.root)
+        @ignore_patterns = Coverband.configuration.ignore
+        @store = options.fetch(:store) { Coverband.configuration.store }
+        @logger = options.fetch(:logger) { Coverband.configuration.logger }
+        @target = options.fetch(:target) { DEFAULT_TARGET }
+
+        @roots = options.fetch(:roots) { Coverband.configuration.all_root_patterns }
+        @roots = @roots.split(',') if @roots.is_a?(String)
+
+        @logged_views = []
+        @views_to_record = []
+      end
+
+      ###
+      # This method is called on every render call, so we try to reduce method calls
+      # and ensure high performance
+      ###
+      def track_views(_name, _start, _finish, _id, payload)
+        if (file = payload[:identifier])
+          if newly_seen_file?(file)
+            logged_views << file
+            views_to_record << file if track_file?(file)
+          end
+        end
+
+        ###
+        # Annoyingly while you get full path for templates
+        # notifications only pass part of the path for layouts dropping any format info
+        # such as .html.erb or .js.erb
+        # http://edgeguides.rubyonrails.org/active_support_instrumentation.html#render_partial-action_view
+        ###
+        return unless (layout_file = payload[:layout])
+        return unless newly_seen_file?(layout_file)
+
+        logged_views << layout_file
+        views_to_record << layout_file if track_file?(layout_file, layout: true)
+      end
+
+      def used_views
+        views = redis_store.smembers(tracker_key)
+        normalized_views = []
+        views.each do |view|
+          roots.each do |root|
+            view = view.gsub(/#{root}/, '')
+          end
+          normalized_views << view
+        end
+        normalized_views
+      end
+
+      def unused_views
+        recently_used_views = used_views
+        all_views = target.reject { |view| recently_used_views.include?(view) }
+        # since layouts don't include format we count them used if they match with ANY formats
+        all_views.reject { |view| view.match(/\/layouts\//) && recently_used_views.any? { |used_view| view.include?(used_view) } }
+      end
+
+      def reset_recordings
+        redis_store.del(tracker_key)
+      end
+
+      def self.supported_version?
+        defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::STRING.split('.').first.to_i >= 4
+      end
+
+      def report_views_tracked
+        views_to_record.each do |file|
+          redis_store.sadd(tracker_key, file)
+        end
+        self.views_to_record = []
+      rescue StandardError => e
+        # we don't want to raise errors if Coverband can't reach redis.
+        # This is a nice to have not a bring the system down
+        logger&.error "Coverband: view_tracker failed to store, error #{e.class.name}"
+      end
+
+      protected
+
+      def newly_seen_file?(file)
+        return false if logged_views.include?(file)
+
+        true
+      end
+
+      def track_file?(file, options = {})
+        @ignore_patterns.none? do |pattern|
+          file.include?(pattern)
+        end && (file.start_with?(@project_directory) || options[:layout])
+      end
+
+      private
+
+      def redis_store
+        store.raw_store
+      end
+
+      def tracker_key
+        'render_tracker'
+      end
+    end
+  end
+end

data/lib/coverband/configuration.rb

@@ -3,15 +3,36 @@
 module Coverband
   class Configuration
     attr_accessor :root_paths, :root,
-                  :ignore, :additional_files, :verbose,
-                  :reporter, :reporting_frequency,
-                  :redis_namespace, :redis_ttl,
-                  :safe_reload_files, :background_reporting_enabled,
+                  :additional_files, :verbose,
+                  :reporter, :redis_namespace, :redis_ttl,
+                  :background_reporting_enabled,
                   :background_reporting_sleep_seconds, :test_env,
-                  :web_enable_clear, :gem_details, :web_debug
+                  :web_enable_clear, :gem_details, :web_debug, :report_on_exit,
+                  :simulate_oneshot_lines_coverage, :track_views, :view_tracker
 
-    attr_writer :logger, :s3_region, :s3_bucket, :s3_access_key_id, :s3_secret_access_key
-    attr_reader :track_gems
+    attr_writer :logger, :s3_region, :s3_bucket, :s3_access_key_id, :s3_secret_access_key, :password
+    attr_reader :track_gems, :ignore, :use_oneshot_lines_coverage
+
+    #####
+    # TODO: This is is brittle and not a great solution to avoid deploy time
+    # actions polluting the 'runtime' metrics
+    #
+    # * should we skip /bin/rails webpacker:compile ?
+    # * Perhaps detect heroku deployment ENV var opposed to tasks?
+    #####
+    IGNORE_TASKS = ['coverband:clear',
+                    'coverband:coverage',
+                    'coverband:coverage_server',
+                    'coverband:migrate']
+
+    # Heroku when building assets runs code from a dynamic directory
+    # /tmp was added to avoid coverage from /tmp/build directories during
+    # heroku asset compilation
+    IGNORE_DEFAULTS = %w[vendor .erb$ .slim$ /tmp internal:prelude schema.rb]
+
+    # Add in missing files which were never loaded
+    # we need to know what all paths to check for unloaded files
+    TRACKED_DEFAULT_PATHS = %w[app lib config]
 
     def initialize
       reset
@@ -20,12 +41,9 @@ module Coverband
     def reset
       @root = Dir.pwd
       @root_paths = []
-      # Heroku when building assets runs code from a dynamic directory
-      # /tmp was added to avoid coverage from /tmp/build directories during
-      # heroku asset compilation
-      @ignore = %w[vendor .erb$ .slim$ /tmp]
+      @ignore = IGNORE_DEFAULTS.dup
+      @seach_paths = TRACKED_DEFAULT_PATHS.dup
       @additional_files = []
-      @reporting_frequency = 0.0
       @verbose = false
       @reporter = 'scov'
       @logger = nil
@@ -36,8 +54,16 @@ module Coverband
       @web_enable_clear = false
       @track_gems = false
       @gem_details = false
+      @track_views = false
       @groups = {}
       @web_debug = false
+      @report_on_exit = true
+      @use_oneshot_lines_coverage = ENV['ONESHOT'] || false
+      @simulate_oneshot_lines_coverage = ENV['SIMULATE_ONESHOT'] || false
+      @current_root = nil
+      @all_root_paths = nil
+      @all_root_patterns = nil
+      @password = nil
 
       # TODO: should we push these to adapter configs
       @s3_region = nil
@@ -45,17 +71,21 @@ module Coverband
       @s3_access_key_id = nil
       @s3_secret_access_key = nil
       @redis_namespace = nil
-      @redis_ttl = nil
+      @redis_ttl = 2_592_000 # in seconds. Default is 30 days.
     end
 
     def logger
-      @logger ||= if defined?(Rails.logger)
+      @logger ||= if defined?(Rails.logger) && Rails.logger
                     Rails.logger
                   else
                     Logger.new(STDOUT)
                   end
     end
 
+    def password
+      @password || ENV['COVERBAND_PASSWORD']
+    end
+
     def s3_bucket
       @s3_bucket || ENV['AWS_BUCKET']
     end
@@ -77,29 +107,47 @@ module Coverband
     end
 
     def store=(store)
-      if store.is_a?(Coverband::Adapters::Base)
-        @store = store
-      else
-        raise 'please pass in an subclass of Coverband::Adapters for supported stores'
-      end
+      raise 'Pass in an instance of Coverband::Adapters' unless store.is_a?(Coverband::Adapters::Base)
+
+      @store = store
+    end
+
+    ###
+    # Search Paths
+    ###
+    def tracked_search_paths
+      "#{Coverband.configuration.current_root}/{#{@seach_paths.join(',')}}/**/*.{rb}"
+    end
+
+    ###
+    # Don't allow the to override defaults
+    ###
+    def search_paths=(path_array)
+      @seach_paths = (@seach_paths + path_array).uniq
+    end
+
+    ###
+    # Don't allow the ignore to override things like gem tracking
+    ###
+    def ignore=(ignored_array)
+      @ignore = (@ignore + ignored_array).uniq
     end
 
     def track_gems=(value)
       @track_gems = value
       return unless @track_gems
+
       # by default we ignore vendor where many deployments put gems
       # we will remove this default if track_gems is set
       @ignore.delete('vendor')
       # while we want to allow vendored gems we don't want to track vendored ruby STDLIB
-      @ignore << 'vendor/ruby-*'
+      @ignore << 'vendor/ruby-*' unless @ignore.include?('vendor/ruby-*')
       add_group('App', root)
       # TODO: rework support for multiple gem paths
-      # currently this supports GEM_HOME (which should be first path)
-      # but various gem managers setup multiple gem paths
-      # gem_paths.each_with_index do |path, index|
-      #   add_group("gems_#{index}", path)
-      # end
-      add_group('Gems', gem_paths.first)
+      # this works but seems hacky and error prone
+      # basically since it is converted to a regex we join all the paths
+      # with a regex 'OR' using '|'
+      add_group('Gems', gem_paths.join('|'))
     end
 
     #
@@ -125,17 +173,23 @@ module Coverband
     end
 
     def current_root
-      File.expand_path(Coverband.configuration.root)
+      @current_root ||= File.expand_path(Coverband.configuration.root).freeze
     end
 
     def all_root_paths
-      roots = Coverband.configuration.root_paths.dup
-      roots += Coverband.configuration.gem_paths.dup if Coverband.configuration.track_gems
-      roots << "#{Coverband.configuration.current_root}/"
-      roots
+      return @all_root_paths if @all_root_paths
+
+      @all_root_paths = Coverband.configuration.root_paths.dup
+      @all_root_paths += Coverband.configuration.gem_paths.dup if Coverband.configuration.track_gems
+      @all_root_paths << "#{Coverband.configuration.current_root}/"
+      @all_root_paths
+    end
+
+    def all_root_patterns
+      @all_root_patterns ||= all_root_paths.map { |path| /^#{path}/ }.freeze
     end
 
-    SKIPPED_SETTINGS = %w(@s3_secret_access_key @store)
+    SKIPPED_SETTINGS = %w[@s3_secret_access_key @store]
     def to_h
       instance_variables
         .each_with_object('gem_paths': gem_paths) do |var, hash|
@@ -143,6 +197,16 @@ module Coverband
         end
     end
 
+    def use_oneshot_lines_coverage=(value)
+      raise(Exception, 'One shot line coverage is only available in ruby >= 2.6') unless one_shot_coverage_implemented_in_ruby_version? || !value
+
+      @use_oneshot_lines_coverage = value
+    end
+
+    def one_shot_coverage_implemented_in_ruby_version?
+      Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.6.0')
+    end
+
     private
 
     def redis_url

data/lib/coverband/integrations/background.rb

@@ -7,6 +7,7 @@ module Coverband
 
     def self.stop
       return unless @thread
+
       @semaphore.synchronize do
         if @thread
           @thread.exit
@@ -16,7 +17,7 @@ module Coverband
     end
 
     def self.running?
-      @thread && @thread.alive?
+      @thread&.alive?
     end
 
     def self.start
@@ -25,12 +26,16 @@ module Coverband
       logger = Coverband.configuration.logger
       @semaphore.synchronize do
         return if running?
-        logger&.debug('Coverband: Starting background reporting')
+
+        logger.debug('Coverband: Starting background reporting') if Coverband.configuration.verbose
         sleep_seconds = Coverband.configuration.background_reporting_sleep_seconds
         @thread = Thread.new do
           loop do
-            Coverband.report_coverage(true)
-            logger&.debug("Coverband: Reported coverage via thread. Sleeping #{sleep_seconds}s") if Coverband.configuration.verbose
+            Coverband.report_coverage
+            Coverband.configuration.view_tracker&.report_views_tracked
+            if Coverband.configuration.verbose
+              logger.debug("Coverband: background reporting coverage (#{Coverband.configuration.store.type}). Sleeping #{sleep_seconds}s")
+            end
             sleep(sleep_seconds)
           end
         end

data/lib/coverband/integrations/{middleware.rb → background_middleware.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Coverband
-  class Middleware
+  class BackgroundMiddleware
     def initialize(app)
       @app = app
     end
@@ -10,11 +10,7 @@ module Coverband
       @app.call(env)
     ensure
       AtExit.register
-      if Coverband.configuration.background_reporting_enabled
-        Background.start
-      else
-        Collectors::Coverage.instance.report_coverage
-      end
+      Background.start if Coverband.configuration.background_reporting_enabled
     end
   end
 end

data/lib/coverband/integrations/bundler.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module BundlerEagerLoad
+  def require(*groups)
+    Coverband.eager_loading_coverage { super }
+  end
+end
+Bundler.singleton_class.prepend BundlerEagerLoad

data/lib/coverband/integrations/report_middleware.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Coverband
+  class ReportMiddleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      @app.call(env)
+    ensure
+      Collectors::Coverage.instance.report_coverage
+    end
+  end
+end

data/lib/coverband/integrations/resque.rb

@@ -1,17 +1,15 @@
 # frozen_string_literal: true
 
-Resque.after_fork do |job|
+Resque.after_fork do |_job|
   Coverband.start
   Coverband.runtime_coverage!
-  # no reason to miss coverage on a first resque job
-  Coverband::Collectors::Delta.set_default_results
 end
 
 Resque.before_first_fork do
   Coverband.eager_loading_coverage!
   Coverband.configuration.background_reporting_enabled = false
   Coverband::Background.stop
-  Coverband::Collectors::Coverage.instance.report_coverage(true)
+  Coverband.report_coverage
 end
 
 module Coverband
@@ -19,7 +17,7 @@ module Coverband
     def perform
       super
     ensure
-      Coverband.report_coverage(true)
+      Coverband.report_coverage
     end
   end
 end

data/lib/coverband/reporters/base.rb

@@ -8,37 +8,61 @@ module Coverband
     ###
     class Base
       class << self
-        include Coverband::Utils::FilePathHelper
+        DATA_KEY = 'data'
+
         def report(store, _options = {})
           all_roots = Coverband.configuration.all_root_paths
           scov_style_report = get_current_scov_data_imp(store, all_roots)
 
-          if Coverband.configuration.verbose
-            msg = "report:\n #{scov_style_report.inspect}"
-            # Coverband.configuration.logger.debug msg
-          end
+          # These are extremelhy verbose but useful during coverband development, not generally for users
+          # Only avaiable by uncommenting this mode is never released
+          # if Coverband.configuration.verbose
+          #   # msg = "report:\n #{scov_style_report.inspect}"
+          #   # Coverband.configuration.logger.debug msg
+          # end
           scov_style_report
         end
 
+        ###
+        # Add back files that exist in the project but have no Coverage
+        # This makes it easy to find and delete files with no references
+        ###
+        def fix_reports(reports)
+          # list all files, even if not tracked by Coverband (0% coverage)
+          tracked_glob = Coverband.configuration.tracked_search_paths
+          filtered_report_files = {}
+
+          reports.each_pair do |report_name, report_data|
+            filtered_report_files[report_name] = {}
+            report_files = Coverband::Utils::Result.add_not_loaded_files(report_data, tracked_glob)
+
+            # apply coverband filters
+            report_files.each_pair do |file, data|
+              next if Coverband.configuration.ignore.any? { |i| file.match(i) }
+
+              filtered_report_files[report_name][file] = data
+            end
+          end
+          filtered_report_files
+        end
+
         protected
 
         def fix_file_names(report_hash, roots)
-          if Coverband.configuration.verbose
-            Coverband.configuration.logger.info "fixing root: #{roots.join(', ')}"
-          end
+          Coverband.configuration.logger.debug "fixing root: #{roots.join(', ')}" if Coverband.configuration.verbose
 
           # normalize names across servers
           report_hash.each_with_object({}) do |(name, report), fixed_report|
             fixed_report[name] = {}
             report.each_pair do |key, vals|
-              filename = relative_path_to_full(key, roots)
-              fixed_report[name][filename] = if fixed_report[name].key?(filename) && fixed_report[name][filename]['data'] && vals['data']
-                                         merged_data = merge_arrays(fixed_report[name][filename]['data'], vals['data'])
-                                         vals['data'] = merged_data
-                                         vals
-                                       else
-                                         vals
-                                       end
+              filename = Coverband::Utils::AbsoluteFileConverter.convert(key)
+              fixed_report[name][filename] = if fixed_report[name].key?(filename) && fixed_report[name][filename][DATA_KEY] && vals[DATA_KEY]
+                                               merged_data = merge_arrays(fixed_report[name][filename][DATA_KEY], vals[DATA_KEY])
+                                               vals[DATA_KEY] = merged_data
+                                               vals
+                                             else
+                                               vals
+                                             end
             end
           end
         end

data/lib/coverband/reporters/console_report.rb

@@ -10,6 +10,7 @@ module Coverband
         scov_style_report = super(store, options)
 
         scov_style_report.each_pair do |file, usage|
+          # TODO: In Coverband 5 deprecate none hash format
           if usage.is_a?(Hash)
             Coverband.configuration.logger.info "#{file}: #{usage['data']}"
           else