coverband 4.2.0 → 4.2.2

data/lib/coverband/adapters/base.rb

@@ -3,45 +3,48 @@
 module Coverband
   module Adapters
     class Base
-      include Coverband::Utils::FilePathHelper
+      DATA_KEY = 'data'
+      FIRST_UPDATED_KEY = 'first_updated_at'
+      LAST_UPDATED_KEY = 'last_updated_at'
+      FILE_HASH = 'file_hash'
+      ABSTRACT_KEY = 'abstract'
+
       attr_accessor :type
 
       def initialize
-        @file_hash_cache = {}
+        @type = Coverband::RUNTIME_TYPE
       end
 
       def clear!
-        raise 'abstract'
+        raise ABSTRACT_KEY
       end
 
-      def clear_file!(_file)
-        raise 'abstract'
+      def clear_file!(_file = nil)
+        raise ABSTRACT_KEY
       end
 
       def migrate!
-        raise 'abstract'
+        raise ABSTRACT_KEY
       end
 
       def size
-        raise 'abstract'
+        raise ABSTRACT_KEY
       end
 
-      def size_in_mib
-        format('%.2f', (size.to_f / 2**20))
+      def save_coverage
+        raise ABSTRACT_KEY
+      end
+
+      def coverage(_local_type = nil)
+        raise ABSTRACT_KEY
       end
 
-      # Note: This could lead to slight race on redis
-      # where multiple processes pull the old coverage and add to it then push
-      # the Coverband 2 had the same issue,
-      # and the tradeoff has always been acceptable
-      def save_report(report)
-        data = report.dup
-        data = merge_reports(data, get_report)
-        save_coverage(data)
+      def size_in_mib
+        format('%.2f', (size.to_f / 2**20))
       end
 
-      def coverage
-        get_report
+      def save_report(_report)
+        raise 'abstract'
       end
 
       def get_coverage_report
@@ -53,60 +56,69 @@ module Coverband
         coverage.keys || []
       end
 
-      # TODO: deprecate / remove?
-      def covered_lines_for_file(file)
-        Array(coverage.dig(file, 'data'))
+      def raw_store
+        raise ABSTRACT_KEY
       end
 
       protected
 
       def split_coverage(types)
         types.reduce({}) do |data, type|
-          data.update(type => get_report(type))
+          if type == Coverband::RUNTIME_TYPE && Coverband.configuration.simulate_oneshot_lines_coverage
+            data.update(type => simulated_runtime_coverage)
+          else
+            data.update(type => coverage(type))
+          end
         end
       end
 
-      def merged_coverage(types)
-        types.reduce({}) do |data, type|
-          merge_reports(data, get_report(type), skip_expansion: true)
+      def simulated_runtime_coverage
+        runtime_data = coverage(Coverband::RUNTIME_TYPE)
+        eager_data = coverage(Coverband::EAGER_TYPE)
+        eager_data.values do |vals|
+          vals['data'].map! { |line_coverage| line_coverage ? (0 - line_coverage) : line_coverage }
         end
+        merge_reports(runtime_data, eager_data, skip_expansion: true)
       end
 
-      def save_coverage
-        raise 'abstract'
-      end
-
-      def get_report
-        raise 'abstract'
+      def merged_coverage(types)
+        types.reduce({}) do |data, type|
+          merge_reports(data, coverage(type), skip_expansion: true)
+        end
       end
 
       def file_hash(file)
-        @file_hash_cache[file] ||= Digest::MD5.file(file).hexdigest
+        Coverband::Utils::FileHasher.hash(file)
       end
 
+      # TODO: modify to extend report inline?
       def expand_report(report)
         expanded = {}
         report_time = Time.now.to_i
-        updated_time = self.type == Coverband::EAGER_TYPE ? nil : report_time
+        updated_time = type == Coverband::EAGER_TYPE ? nil : report_time
         report.each_pair do |key, line_data|
           extended_data = {
-            'first_updated_at' => report_time,
-            'last_updated_at' => updated_time,
-            'file_hash' => file_hash(key),
-            'data' => line_data
+            FIRST_UPDATED_KEY => report_time,
+            LAST_UPDATED_KEY => updated_time,
+            FILE_HASH => file_hash(key),
+            DATA_KEY => line_data
           }
-          expanded[full_path_to_relative(key)] = extended_data
+          expanded[Utils::RelativeFileConverter.convert(key)] = extended_data
         end
         expanded
       end
 
       def merge_reports(new_report, old_report, options = {})
+        # transparently update from RUNTIME_TYPE = nil to RUNTIME_TYPE = :runtime
+        # transparent update for format coveband_3_2
+        old_report = coverage(nil, override_type: nil) if old_report.nil? && type == Coverband::RUNTIME_TYPE
+
         new_report = expand_report(new_report) unless options[:skip_expansion]
         keys = (new_report.keys + old_report.keys).uniq
         keys.each do |file|
           new_report[file] = if new_report[file] &&
                                 old_report[file] &&
-                                new_report[file]['file_hash'] == old_report[file]['file_hash']
+                                new_report[file][FILE_HASH] == old_report[file][FILE_HASH]
                                merge_expanded_data(new_report[file], old_report[file])
                              elsif new_report[file]
                                new_report[file]
@@ -119,15 +131,22 @@ module Coverband
 
       def merge_expanded_data(new_expanded, old_expanded)
         {
-          'first_updated_at' => old_expanded['first_updated_at'],
-          'last_updated_at' => new_expanded['last_updated_at'],
-          'file_hash' => new_expanded['file_hash'],
-          'data' => array_add(new_expanded['data'], old_expanded['data'])
+          FIRST_UPDATED_KEY => old_expanded[FIRST_UPDATED_KEY],
+          LAST_UPDATED_KEY => new_expanded[LAST_UPDATED_KEY],
+          FILE_HASH => new_expanded[FILE_HASH],
+          DATA_KEY => array_add(new_expanded[DATA_KEY], old_expanded[DATA_KEY])
         }
       end
 
+      # TODO: This should only be 2 cases get our dup / not dups aligned
       def array_add(latest, original)
-        latest.map.with_index { |v, i| (v && original[i]) ? v + original[i] : nil }
+        if Coverband.configuration.use_oneshot_lines_coverage
+          latest.map!.with_index { |v, i| (v + original[i] >= 1 ? 1 : 0) if v && original[i] }
+        elsif Coverband.configuration.simulate_oneshot_lines_coverage
+          latest.map.with_index { |v, i| (v + original[i] >= 1 ? 1 : 0) if v && original[i] }
+        else
+          latest.map.with_index { |v, i| (v && original[i]) ? v + original[i] : nil }
+        end
       end
     end
   end

data/lib/coverband/adapters/file_store.rb

@@ -28,21 +28,27 @@ module Coverband
         raise NotImplementedError, "FileStore doesn't support migrations"
       end
 
-      private
-
-      attr_accessor :path
-
-      def save_coverage(report)
-        File.open(path, 'w') { |f| f.write(report.to_json) }
-      end
-
-      def get_report
+      def coverage(_local_type = nil)
         if File.exist?(path)
           JSON.parse(File.read(path))
         else
           {}
         end
       end
+
+      def save_report(report)
+        data = report.dup
+        data = merge_reports(data, coverage)
+        File.open(path, 'w') { |f| f.write(data.to_json) }
+      end
+
+      def raw_store
+        raise NotImplementedError, "FileStore doesn't support raw_store"
+      end
+
+      private
+
+      attr_accessor :path
     end
   end
 end

data/lib/coverband/adapters/hash_redis_store.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Coverband
+  module Adapters
+    class HashRedisStore < Base
+      FILE_KEY = 'file'
+      FILE_LENGTH_KEY = 'file_length'
+      META_DATA_KEYS = [DATA_KEY, FIRST_UPDATED_KEY, LAST_UPDATED_KEY].freeze
+      ###
+      # This key isn't related to the coverband version, but to the interal format
+      # used to store data to redis. It is changed only when breaking changes to our
+      # redis format are required.
+      ###
+      REDIS_STORAGE_FORMAT_VERSION = 'coverband_hash_3_3'
+
+      attr_reader :redis_namespace
+
+      def initialize(redis, opts = {})
+        super()
+        @redis_namespace = opts[:redis_namespace]
+        @format_version = REDIS_STORAGE_FORMAT_VERSION
+        @redis = redis
+        raise 'HashRedisStore requires redis >= 2.6.0' unless supported?
+
+        @ttl = opts[:ttl] || -1
+        @relative_file_converter = opts[:relative_file_converter] || Utils::RelativeFileConverter
+      end
+
+      def supported?
+        Gem::Version.new(@redis.info['redis_version']) >= Gem::Version.new('2.6.0')
+      end
+
+      def clear!
+        old_type = type
+        Coverband::TYPES.each do |type|
+          self.type = type
+          file_keys = files_set
+          @redis.del(*file_keys) if file_keys.any?
+          @redis.del(files_key)
+        end
+        self.type = old_type
+      end
+
+      def clear_file!(file)
+        file_hash = file_hash(file)
+        relative_path_file = @relative_file_converter.convert(file)
+        Coverband::TYPES.each do |type|
+          @redis.del(key(relative_path_file, type, file_hash: file_hash))
+        end
+        @redis.srem(files_key, relative_path_file)
+      end
+
+      def save_report(report)
+        report_time = Time.now.to_i
+        updated_time = type == Coverband::EAGER_TYPE ? nil : report_time
+        script_id = hash_incr_script
+        @redis.pipelined do
+          keys = report.map do |file, data|
+            relative_file = @relative_file_converter.convert(file)
+            file_hash = file_hash(relative_file)
+            key = key(relative_file, file_hash: file_hash)
+            script_input = save_report_script_input(
+              key: key,
+              file: relative_file,
+              file_hash: file_hash,
+              data: data,
+              report_time: report_time,
+              updated_time: updated_time
+            )
+            @redis.evalsha(script_id, script_input[:keys], script_input[:args])
+            key
+          end
+          @redis.sadd(files_key, keys) if keys.any?
+        end
+      end
+
+      def coverage(local_type = nil)
+        files_set(local_type).each_with_object({}) do |key, hash|
+          add_coverage_for_file(key, hash)
+        end
+      end
+
+      def raw_store
+        @redis
+      end
+
+      def size
+        'not available'
+      end
+
+      def size_in_mib
+        'not available'
+      end
+
+      private
+
+      def add_coverage_for_file(key, hash)
+        data_from_redis = @redis.hgetall(key)
+
+        return if data_from_redis.empty?
+
+        file = data_from_redis[FILE_KEY]
+        return unless file_hash(file) == data_from_redis[FILE_HASH]
+
+        data = coverage_data_from_redis(data_from_redis)
+        hash[file] = data_from_redis.select { |meta_data_key, _value| META_DATA_KEYS.include?(meta_data_key) }.merge!('data' => data)
+        hash[file].merge!(LAST_UPDATED_KEY => hash[file][LAST_UPDATED_KEY].to_i, FIRST_UPDATED_KEY => hash[file][FIRST_UPDATED_KEY].to_i)
+      end
+
+      def coverage_data_from_redis(data_from_redis)
+        max = data_from_redis[FILE_LENGTH_KEY].to_i - 1
+        Array.new(max + 1) do |index|
+          line_coverage = data_from_redis[index.to_s]
+          line_coverage.nil? ? nil : line_coverage.to_i
+        end
+      end
+
+      def save_report_script_input(key:, file:, file_hash:, data:, report_time:, updated_time:)
+        data.each_with_index
+            .each_with_object(keys: [key], args: [report_time, updated_time, file, file_hash, @ttl, data.length]) do |(coverage, index), hash|
+          if coverage
+            hash[:keys] << index
+            hash[:args] << coverage
+          end
+        end
+      end
+
+      def hash_incr_script
+        @hash_incr_script ||= @redis.script(:load, <<~LUA)
+          local first_updated_at = table.remove(ARGV, 1)
+          local last_updated_at = table.remove(ARGV, 1)
+          local file = table.remove(ARGV, 1)
+          local file_hash = table.remove(ARGV, 1)
+          local ttl = table.remove(ARGV, 1)
+          local file_length = table.remove(ARGV, 1)
+          local hash_key = table.remove(KEYS, 1)
+          redis.call('HMSET', hash_key, '#{LAST_UPDATED_KEY}', last_updated_at, '#{FILE_KEY}', file, '#{FILE_HASH}', file_hash, '#{FILE_LENGTH_KEY}', file_length)
+          redis.call('HSETNX', hash_key, '#{FIRST_UPDATED_KEY}', first_updated_at)
+          for i, key in ipairs(KEYS) do
+            if ARGV[i] == '-1' then
+              redis.call("HSET", hash_key, key, ARGV[i])
+            else
+              redis.call("HINCRBY", hash_key, key, ARGV[i])
+            end
+          end
+          if ttl ~= '-1' then
+            redis.call("EXPIRE", hash_key, ttl)
+          end
+        LUA
+      end
+
+      def values_from_redis(local_type, files)
+        return files if files.empty?
+
+        @redis.mget(*files.map { |file| key(file, local_type) }).map do |value|
+          value.nil? ? {} : JSON.parse(value)
+        end
+      end
+
+      def relative_paths(files)
+        files&.map! { |file| full_path_to_relative(file) }
+      end
+
+      def files_set(local_type = nil)
+        @redis.smembers(files_key(local_type))
+      end
+
+      def files_key(local_type = nil)
+        "#{key_prefix(local_type)}.files"
+      end
+
+      def key(file, local_type = nil, file_hash:)
+        [key_prefix(local_type), file, file_hash].join('.')
+      end
+
+      def key_prefix(local_type = nil)
+        local_type ||= type
+        [@format_version, @redis_namespace, local_type].compact.join('.')
+      end
+    end
+  end
+end

data/lib/coverband/adapters/redis_store.rb

@@ -13,23 +13,31 @@ module Coverband
       ###
       REDIS_STORAGE_FORMAT_VERSION = 'coverband_3_2'
 
+      attr_reader :redis_namespace
+
       def initialize(redis, opts = {})
         super()
         @redis           = redis
         @ttl             = opts[:ttl]
         @redis_namespace = opts[:redis_namespace]
         @format_version  = REDIS_STORAGE_FORMAT_VERSION
+        @keys            = {}
+        Coverband::TYPES.each do |type|
+          @keys[type] = [@format_version, @redis_namespace, type].compact.join('.')
+        end
       end
 
       def clear!
         Coverband::TYPES.each do |type|
           @redis.del(type_base_key(type))
         end
+        # temporarily clear the old namespace of coverband_3_2
+        @redis.del(type_base_key(nil))
       end
 
       def clear_file!(filename)
         Coverband::TYPES.each do |type|
-          data = get_report(type)
+          data = coverage(type)
           data.delete(filename)
           save_coverage(data, type)
         end
@@ -47,18 +55,18 @@ module Coverband
       def migrate!
         reset_base_key
         @format_version = 'coverband3_1'
-        previous_data = get_report
+        previous_data = coverage
         if previous_data.empty?
           puts 'no previous data to migrate found'
           exit 0
         end
         relative_path_report = previous_data.each_with_object({}) do |(key, vals), fixed_report|
-          fixed_report[full_path_to_relative(key)] = vals
+          fixed_report[Utils::RelativeFileConverter.convert(key)] = vals
         end
         clear!
         reset_base_key
         @format_version = REDIS_STORAGE_FORMAT_VERSION
-        save_coverage(merge_reports(get_report, relative_path_report, skip_expansion: true))
+        save_coverage(merge_reports(coverage, relative_path_report, skip_expansion: true))
       end
 
       def type=(type)
@@ -66,6 +74,26 @@ module Coverband
         reset_base_key
       end
 
+      def coverage(local_type = nil, opts = {})
+        local_type ||= opts.key?(:override_type) ? opts[:override_type] : type
+        data = redis.get type_base_key(local_type)
+        data ? JSON.parse(data) : {}
+      end
+
+      # Note: This could lead to slight race on redis
+      # where multiple processes pull the old coverage and add to it then push
+      # the Coverband 2 had the same issue,
+      # and the tradeoff has always been acceptable
+      def save_report(report)
+        data = report.dup
+        data = merge_reports(data, coverage)
+        save_coverage(data)
+      end
+
+      def raw_store
+        @redis
+      end
+
       private
 
       attr_reader :redis
@@ -79,7 +107,7 @@ module Coverband
       end
 
       def type_base_key(local_type)
-        [@format_version, @redis_namespace, local_type].compact.join('.')
+        @keys[local_type]
       end
 
       def save_coverage(data, local_type = nil)
@@ -87,12 +115,6 @@ module Coverband
         redis.set type_base_key(local_type), data.to_json
         redis.expire(type_base_key(local_type), @ttl) if @ttl
       end
-
-      def get_report(local_type = nil)
-        local_type ||= type
-        data = redis.get type_base_key(local_type)
-        data ? JSON.parse(data) : {}
-      end
     end
   end
 end

data/lib/coverband/at_exit.rb

@@ -16,19 +16,10 @@ module Coverband
         at_exit do
           ::Coverband::Background.stop
 
-          #####
-          # TODO: This is is brittle and not a great solution to avoid deploy time
-          # actions polluting the 'runtime' metrics
-          #
-          # * should we skip /bin/rails webpacker:compile ?
-          # * Perhaps detect heroku deployment ENV var opposed to tasks?
-          #####
-          default_heroku_tasks = ['assets:clean', 'assets:precompile']
-          if defined?(Rake) && Rake.respond_to?(:application) && (Rake&.application&.top_level_tasks || []).any? { |task| default_heroku_tasks.include?(task) }
+          if !Coverband.configuration.report_on_exit
             # skip reporting
           else
-            Coverband.report_coverage(true)
-            #Coverband.configuration.logger&.debug('Coverband: Reported coverage before exit')
+            Coverband.report_coverage
           end
         end
       end

data/lib/coverband/collectors/coverage.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'singleton'
 require_relative 'delta'
 
@@ -13,13 +14,16 @@ module Coverband
     class Coverage
       include Singleton
 
+      def self.ruby_version_greater_than_or_equal_to?(version)
+        Gem::Version.new(RUBY_VERSION) >= Gem::Version.new(version)
+      end
+
       def reset_instance
         @project_directory = File.expand_path(Coverband.configuration.root)
-        @ignore_patterns = Coverband.configuration.ignore + ['internal:prelude', 'schema.rb']
-        @reporting_frequency = Coverband.configuration.reporting_frequency
+        @ignore_patterns = Coverband.configuration.ignore
         @store = Coverband.configuration.store
-        @verbose  = Coverband.configuration.verbose
-        @logger   = Coverband.configuration.logger
+        @verbose = Coverband.configuration.verbose
+        @logger = Coverband.configuration.logger
         @test_env = Coverband.configuration.test_env
         @track_gems = Coverband.configuration.track_gems
         Delta.reset
@@ -27,36 +31,34 @@ module Coverband
       end
 
       def runtime!
-        @store.type = nil
+        @store.type = Coverband::RUNTIME_TYPE
       end
 
       def eager_loading!
         @store.type = Coverband::EAGER_TYPE
       end
 
-      def report_coverage(force_report = false)
-        return if !ready_to_report? && !force_report
-        raise 'no Coverband store set' unless @store
+      def eager_loading
+        old_coverage_type = @store.type
+        eager_loading!
+        yield
+      ensure
+        report_coverage
+        @store.type = old_coverage_type
+      end
 
-        original_previous_set = Delta.previous_results
-        files_with_line_usage = filtered_files(Delta.results)
+      def report_coverage
+        @semaphore.synchronize do
+          raise 'no Coverband store set' unless @store
 
-        ###
-        # Hack to prevent processes and threads from reporting first Coverage hit
-        # when we are in runtime collection mode, which do not have a cache of previous
-        # coverage to remove the initial stdlib Coverage loading data
-        ###
-        if ((original_previous_set.nil? && @store.type == Coverband::EAGER_TYPE) ||
-           (original_previous_set && @store.type != Coverband::EAGER_TYPE))
+          files_with_line_usage = filtered_files(Delta.results)
           @store.save_report(files_with_line_usage)
         end
-      rescue StandardError => err
-        if @verbose
-          @logger&.error 'coverage failed to store'
-          @logger&.error "error: #{err.inspect} #{err.message}"
-          @logger&.error err.backtrace
-        end
-        raise err if @test_env
+      rescue StandardError => e
+        @logger&.error 'coverage failed to store'
+        @logger&.error "Coverband Error: #{e.inspect} #{e.message}"
+        e.backtrace.each { |line| @logger&.error line } if @verbose
+        raise e if @test_env
       end
 
       protected
@@ -78,30 +80,22 @@ module Coverband
       private
 
       def filtered_files(new_results)
-        new_results.each_with_object({}) do |(file, line_counts), file_line_usage|
-          file_line_usage[file] = line_counts if track_file?(file)
-        end.select { |_file_name, coverage| coverage.any? { |value| value&.nonzero? } }
-      end
-
-      def ready_to_report?
-        (rand * 100.0) >= (100.0 - @reporting_frequency)
+        new_results.select! { |file, coverage| track_file?(file) && coverage.any? { |value| value&.nonzero? } }
+        new_results
       end
 
       def initialize
-        if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3.0')
-          raise NotImplementedError, 'not supported until Ruby 2.3.0 and later'
-        end
+        @semaphore = Mutex.new
+        raise NotImplementedError, 'Coverage needs Ruby > 2.3.0' if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3.0')
+
         require 'coverage'
-        if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.5.0')
+        if Coverage.ruby_version_greater_than_or_equal_to?('2.6.0')
+          ::Coverage.start(oneshot_lines: Coverband.configuration.use_oneshot_lines_coverage) unless ::Coverage.running?
+        elsif Coverage.ruby_version_greater_than_or_equal_to?('2.5.0')
           ::Coverage.start unless ::Coverage.running?
         else
           ::Coverage.start
         end
-        if Coverband.configuration.safe_reload_files
-          Coverband.configuration.safe_reload_files.each do |safe_file|
-            load safe_file
-          end
-        end
         reset_instance
       end
     end