coverband 4.2.0 → 4.2.2

data/test/benchmarks/benchmark.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 namespace :benchmarks do
   # https://github.com/evanphx/benchmark-ips
   # Enable and start GC before each job run. Disable GC afterwards.
@@ -13,11 +14,9 @@ namespace :benchmarks do
       run_gc
     end
 
-    def warmup_stats(*)
-    end
+    def warmup_stats(*); end
 
-    def add_report(*)
-    end
+    def add_report(*); end
 
     private
 
@@ -28,37 +27,14 @@ namespace :benchmarks do
     end
   end
 
-  def classifier_dir
-    File.join(File.dirname(__FILE__), 'classifier-reborn')
-  end
-
-  def clone_classifier
-    # rubocop:disable Style/IfUnlessModifier
-    unless Dir.exist? classifier_dir
-      system "git clone https://github.com/jekyll/classifier-reborn.git #{classifier_dir}"
-    end
-    # rubocop:enable Style/IfUnlessModifier
-  end
-
   # desc 'setup standard benchmark'
   task :setup do
-    clone_classifier
-    $LOAD_PATH.unshift(File.join(classifier_dir, 'lib'))
     require 'benchmark'
     require 'benchmark/ips'
 
-    # TODO: ok this is interesting and weird
-    # basically the earlier I require coverage and
-    # then require files the larger perf impact
-    # this is somewhat expected but can lead to significant perf diffs
-    # for example moving `require 'classifier-reborn'` below the coverage.start
-    # results in 1.5x slower vs "difference falls within error"
-    # moving from 5 second of time to 12 still shows slower based on when classifier is required
-    # make sure to be plugged in while benchmarking ;) Otherwise you get very unreliable results
-    require 'classifier-reborn'
-    if ENV['COVERAGE']
+    if ENV['COVERAGE'] || ENV['ONESHOT']
       require 'coverage'
-      ::Coverage.start
+      ::Coverage.start(oneshot_lines: !!ENV['ONESHOT'])
     end
     require 'redis'
     require 'coverband'
@@ -79,9 +55,10 @@ namespace :benchmarks do
   task :setup_redis do
     Coverband.configure do |config|
       config.root                = Dir.pwd
-      config.reporting_frequency = 100.0
       config.logger              = $stdout
       config.store               = benchmark_redis_store
+      config.use_oneshot_lines_coverage = true if ENV['ONESHOT']
+      config.simulate_oneshot_lines_coverage = true if ENV['SIMULATE_ONESHOT']
     end
   end
 
@@ -89,39 +66,13 @@ namespace :benchmarks do
   task :setup_file do
     Coverband.configure do |config|
       config.root                = Dir.pwd
-      config.reporting_frequency = 100.0
       config.logger              = $stdout
       file_path                  = '/tmp/benchmark_store.json'
       config.store               = Coverband::Adapters::FileStore.new(file_path)
     end
   end
 
-  def bayes_classification
-    b = ClassifierReborn::Bayes.new 'Interesting', 'Uninteresting'
-    b.train_interesting 'here are some good words. I hope you love them'
-    b.train_uninteresting 'here are some bad words, I hate you'
-    b.classify 'I hate bad words and you' # returns 'Uninteresting'
-  end
-
-  def lsi_classification
-    lsi = ClassifierReborn::LSI.new
-    strings = [['This text deals with dogs. Dogs.', :dog],
-               ['This text involves dogs too. Dogs! ', :dog],
-               ['This text revolves around cats. Cats.', :cat],
-               ['This text also involves cats. Cats!', :cat],
-               ['This text involves birds. Birds.', :bird]]
-    strings.each { |x| lsi.add_item x.first, x.last }
-    lsi.search('dog', 3)
-    lsi.find_related(strings[2], 2)
-    lsi.classify 'This text is also about dogs!'
-  end
-
   def work
-    5.times do
-      bayes_classification
-      lsi_classification
-    end
-
     # simulate many calls to the same line
     10_000.times { Dog.new.bark }
   end
@@ -134,7 +85,7 @@ namespace :benchmarks do
       x.config(time: 12, warmup: 5, suite: suite)
       x.report 'coverband' do
         work
-        Coverband.report_coverage(true)
+        Coverband.report_coverage
       end
       x.report 'no coverband' do
         work
@@ -145,9 +96,11 @@ namespace :benchmarks do
     Coverband::Collectors::Coverage.instance.reset_instance
   end
 
+  LINES = 45
+  NON_NIL_LINES = 18
   def fake_line_numbers
-    24.times.each_with_object({}) do |line, line_hash|
-      line_hash[(line + 1).to_s] = rand(5)
+    LINES.times.map do |line|
+      coverage = (line < NON_NIL_LINES) ? rand(5) : nil
     end
   end
 
@@ -160,6 +113,7 @@ namespace :benchmarks do
   def adjust_report(report)
     report.keys.each do |file|
       next unless rand < 0.15
+
       report[file] = fake_line_numbers
     end
     report
@@ -169,12 +123,8 @@ namespace :benchmarks do
     ###
     # this is a hack because in the benchmark we don't have real files
     ###
-    def store.file_hash(file)
-      if @file_hash_cache[file]
-        @file_hash_cache[file]
-      else
-        @file_hash_cache[file] = Digest::MD5.file(__FILE__).hexdigest
-      end
+    def store.file_hash(_file)
+      @file_hash ||= Digest::MD5.file(__FILE__).hexdigest
     end
 
     def store.full_path_to_relative(file)
@@ -191,7 +141,13 @@ namespace :benchmarks do
     5.times { store.save_report(report) }
     Benchmark.ips do |x|
       x.config(time: 15, warmup: 5)
-      x.report('store_reports') { store.save_report(report) }
+      x.report('store_reports_all') { store.save_report(report) }
+    end
+    keys_subset = report.keys.first(100)
+    report_subset = report.select { |key, _value| keys_subset.include?(key) }
+    Benchmark.ips do |x|
+      x.config(time: 20, warmup: 5)
+      x.report('store_reports_subset') { store.save_report(report_subset) }
     end
   end
 
@@ -214,11 +170,54 @@ namespace :benchmarks do
     end.pretty_print
     data = $stdout.string
     $stdout = previous_out
-    raise 'leaking memory!!!' unless data.match('Total retained:  0 bytes')
+    unless data.match('Total retained:  0 bytes')
+      puts data
+      raise 'leaking memory!!!'
+    end
+  ensure
+    $stdout = previous_out
+  end
+
+  def measure_memory_report_coverage
+    require 'memory_profiler'
+    report = fake_report
+    store = benchmark_redis_store
+    store.clear!
+    mock_files(store)
+
+    # warmup
+    3.times { Coverband.report_coverage }
+
+    previous_out = $stdout
+    capture = StringIO.new
+    $stdout = capture
+
+    MemoryProfiler.report do
+      10.times do
+        Coverband.report_coverage
+        ###
+        # Set to nil not {} as it is easier to verify that no memory is retained when nil gets released
+        # don't use Coverband::Collectors::Delta.reset which sets to {}
+        # we clear this as this one variable is expected to retain memory and is a false positive
+        ###
+        Coverband::Collectors::Delta.class_variable_set(:@@previous_coverage, nil)
+      end
+    end.pretty_print
+    data = $stdout.string
+    $stdout = previous_out
+    unless data.match('Total retained:  0 bytes')
+      puts data
+      raise 'leaking memory!!!'
+    end
   ensure
     $stdout = previous_out
   end
 
+  ###
+  # TODO: This currently fails, as it holds a string in redis adapter
+  # but really Coverband shouldn't be configured multiple times and the leak is small
+  # not including in test suite but we can try to figure it out and fix.
+  ###
   def measure_configure_memory
     require 'memory_profiler'
     # warmup
@@ -232,24 +231,79 @@ namespace :benchmarks do
       10.times do
         Coverband.configure do |config|
           redis_url = ENV['CACHE_REDIS_URL'] || ENV['REDIS_URL']
-          config.store = Coverband::Adapters::RedisStore.new(Redis.new(url: redis_url), redis_namespace: 'coverband_data')
+          config.store = Coverband::Adapters::RedisStore.new(Redis.new(url: redis_url), redis_namespace: 'coverband_bench_data')
         end
       end
     end.pretty_print
     data = $stdout.string
     $stdout = previous_out
-    puts data
-    raise 'leaking memory!!!' unless data.match('Total retained:  0 bytes')
+    unless data.match('Total retained:  0 bytes')
+      puts data
+      raise 'leaking memory!!!'
+    end
   ensure
     $stdout = previous_out
   end
 
+  desc 'checks memory of collector'
+  task memory_check: [:setup] do
+    require 'pry-byebug'
+    require 'objspace'
+    puts 'memory load check'
+    puts(ObjectSpace.memsize_of_all / 2**20)
+    data = File.read('./tmp/debug_data.json')
+    # about 2mb
+    puts(ObjectSpace.memsize_of(data) / 2**20)
+
+    json_data = JSON.parse(data)
+    # this seems to just show the value of the pointer
+    # puts(ObjectSpace.memsize_of(json_data) / 2**20)
+    # implies json takes 10-12 mb
+    puts(ObjectSpace.memsize_of_all / 2**20)
+
+    json_data = nil
+    GC.start
+    json_data = JSON.parse(data)
+    # this seems to just show the value of the pointer
+    # puts(ObjectSpace.memsize_of(json_data) / 2**20)
+    # implies json takes 10-12 mb
+    puts(ObjectSpace.memsize_of_all / 2**20)
+
+    json_data = nil
+    GC.start
+    json_data = JSON.parse(data)
+    # this seems to just show the value of the pointer
+    # puts(ObjectSpace.memsize_of(json_data) / 2**20)
+    # implies json takes 10-12 mb
+    puts(ObjectSpace.memsize_of_all / 2**20)
+
+    json_data = nil
+    GC.start
+    json_data = JSON.parse(data)
+    # this seems to just show the value of the pointer
+    # puts(ObjectSpace.memsize_of(json_data) / 2**20)
+    # implies json takes 10-12 mb
+    puts(ObjectSpace.memsize_of_all / 2**20)
+
+    json_data = nil
+    GC.start
+    puts(ObjectSpace.memsize_of_all / 2**20)
+    debugger
+    puts 'done'
+  end
+
   desc 'runs memory reporting on Redis store'
   task memory_reporting: [:setup] do
     puts 'runs memory benchmarking to ensure we dont leak'
     measure_memory
   end
 
+  desc 'runs memory reporting on report_coverage'
+  task memory_reporting_report_coverage: [:setup] do
+    puts 'runs memory benchmarking on report_coverage to ensure we dont leak'
+    measure_memory_report_coverage
+  end
+
   desc 'runs memory reporting on configure'
   task memory_configure_reporting: [:setup] do
     puts 'runs memory benchmarking on configure to ensure we dont leak'
@@ -259,7 +313,12 @@ namespace :benchmarks do
   desc 'runs memory leak check via Rails tests'
   task memory_rails: [:setup] do
     puts 'runs memory rails test to ensure we dont leak'
-    puts `COVERBAND_MEMORY_TEST=true bundle exec m test/unit/rails_full_stack_test.rb:22`
+    puts `COVERBAND_MEMORY_TEST=true bundle exec test/forked/rails_full_stack_test.rb`
+  end
+
+  desc 'runs memory leak checks'
+  task memory: %i[memory_reporting memory_reporting_report_coverage memory_rails] do
+    puts 'done'
   end
 
   desc 'runs benchmarks on reporting large sets of files to redis'
@@ -269,13 +328,36 @@ namespace :benchmarks do
   end
 
   # desc 'runs benchmarks on default redis setup'
-  task run_redis: [:setup, :setup_redis] do
+  task run_redis: %i[setup setup_redis] do
     puts 'Coverband configured with default Redis store'
     run_work(true)
   end
 
+  def run_big
+    require 'memory_profiler'
+    require './test/unique_files'
+
+    4000.times { |index| require_unique_file('dog.rb.erb', dog_number: index) }
+    # warmup
+    3.times { Coverband.report_coverage }
+    dogs = 400.times.map { |index| Object.const_get("Dog#{index}") }
+    MemoryProfiler.report do
+      10.times do
+        dogs.each(&:bark)
+        Coverband.report_coverage
+      end
+    end.pretty_print
+  end
+
+  task run_big: %i[setup setup_redis] do
+    # ensure we cleared from last run
+    benchmark_redis_store.clear!
+
+    run_big
+  end
+
   # desc 'runs benchmarks file store'
-  task run_file: [:setup, :setup_file] do
+  task run_file: %i[setup setup_file] do
     puts 'Coverband configured with file store'
     run_work(true)
   end
@@ -297,6 +379,18 @@ namespace :benchmarks do
     `open tmp/timeseries.jpg`
   end
 
+  desc 'benchmark initialization of rails'
+  task :init_rails do
+    require 'benchmark'
+    require 'benchmark/ips'
+    Benchmark.ips do |x|
+      x.config(time: 60, warmup: 0)
+      x.report('init_rails') do
+        system('bundle exec rake init_rails -f ./test/benchmarks/init_rails.rake')
+      end
+    end
+  end
+
   desc 'compare Coverband Ruby Coverage with Filestore with normal Ruby'
   task :compare_file do
     puts 'comparing Coverage loaded/not, this takes some time for output...'

data/test/benchmarks/init_rails.rake

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'rails'
+require 'coverband'
+
+desc 'Initialize rails'
+task 'init_rails' do
+  Coverband.configure("./test/rails#{Rails::VERSION::MAJOR}_dummy/config/coverband.rb")
+  require "./test/rails#{Rails::VERSION::MAJOR}_dummy/config/environment"
+end

data/test/coverband/adapters/base_test.rb

@@ -3,49 +3,80 @@
 require File.expand_path('../../test_helper', File.dirname(__FILE__))
 
 class AdaptersBaseTest < Minitest::Test
-  def setup
-    super
-    @test_file_path = '/tmp/coverband_filestore_test_path.json'
-    @store = Coverband::Adapters::FileStore.new(@test_file_path)
-    mock_file_hash
+
+  def test_abstract_methods
+    abstract_methods = %w(clear! clear_file! migrate! size save_coverage coverage)
+    abstract_methods.each do |method|
+      assert_raises RuntimeError do
+        Coverband::Adapters::Base.new.send(method.to_sym)
+      end
+    end
+  end
+
+  def test_size_in_mib
+    base = Coverband::Adapters::Base.new
+    def base.size
+      3.0
+    end
+    assert_equal "0.00", base.size_in_mib
   end
 
-  def test_covered_merge
-    old_time = 1541958097
-    current_time = Time.now.to_i
-    old_data = {
-      'first_updated_at' => old_time,
-      'last_updated_at' => current_time,
-      'file_hash' => 'abcd',
-      'data' => [5, 7, nil]
-    }
-    old_report = { '/projects/coverband_demo/config/coverband.rb' => old_data,
-                   '/projects/coverband_demo/config/initializers/assets.rb' => old_data,
-                   '/projects/coverband_demo/config/initializers/cookies_serializer.rb' => old_data }
-    new_report = { '/projects/coverband_demo/config/coverband.rb' => [5, 7, nil],
-                   '/projects/coverband_demo/config/initializers/filter_logging.rb' => [5, 7, nil],
-                   '/projects/coverband_demo/config/initializers/wrap_parameters.rb' => [5, 7, nil],
-                   '/projects/coverband_demo/app/controllers/application_controller.rb' => [5, 7, nil] }
-    expected_merge = {
-      'first_updated_at' => old_time,
-      'last_updated_at' => current_time,
-      'file_hash' => 'abcd',
-      'data' => [10, 14, nil]
-    }
-    new_data = {
-      'first_updated_at' => current_time,
-      'last_updated_at' => current_time,
-      'file_hash' => 'abcd',
-      'data' => [5, 7, nil]
-    }
-    expected_result = {
-      '/projects/coverband_demo/app/controllers/application_controller.rb' => new_data,
-      '/projects/coverband_demo/config/coverband.rb' => expected_merge,
-      '/projects/coverband_demo/config/initializers/assets.rb' => old_data,
-      '/projects/coverband_demo/config/initializers/cookies_serializer.rb' => old_data,
-      '/projects/coverband_demo/config/initializers/filter_logging.rb' => new_data,
-      '/projects/coverband_demo/config/initializers/wrap_parameters.rb' => new_data
-    }
-    assert_equal expected_result, @store.send(:merge_reports, new_report, old_report)
+  def test_array_add
+    original = [5, 7, nil, nil]
+    latest = [3, 4, nil, 1]
+    assert_equal [8, 11, nil, nil], Coverband::Adapters::Base.new.send(:array_add, latest, original)
+    Coverband.configuration.stubs(:use_oneshot_lines_coverage).returns(true)
+    assert_equal [1, 1, nil, nil], Coverband::Adapters::Base.new.send(:array_add, latest, original)
+    Coverband.configuration.stubs(:use_oneshot_lines_coverage).returns(false)
+    Coverband.configuration.stubs(:simulate_oneshot_lines_coverage).returns(true)
+    assert_equal [1, 1, nil, nil], Coverband::Adapters::Base.new.send(:array_add, latest, original)
+  end
+
+  describe 'Coverband::Adapters::Base using file' do
+    def setup
+      super
+      @test_file_path = '/tmp/coverband_filestore_test_path.json'
+      @store = Coverband::Adapters::FileStore.new(@test_file_path)
+      mock_file_hash
+    end
+
+    def test_covered_merge
+      old_time = 1541958097
+      current_time = Time.now.to_i
+      old_data = {
+        'first_updated_at' => old_time,
+        'last_updated_at' => current_time,
+        'file_hash' => 'abcd',
+        'data' => [5, 7, nil]
+      }
+      old_report = { '/projects/coverband_demo/config/coverband.rb' => old_data,
+                     '/projects/coverband_demo/config/initializers/assets.rb' => old_data,
+                     '/projects/coverband_demo/config/initializers/cookies_serializer.rb' => old_data }
+      new_report = { '/projects/coverband_demo/config/coverband.rb' => [5, 7, nil],
+                     '/projects/coverband_demo/config/initializers/filter_logging.rb' => [5, 7, nil],
+                     '/projects/coverband_demo/config/initializers/wrap_parameters.rb' => [5, 7, nil],
+                     '/projects/coverband_demo/app/controllers/application_controller.rb' => [5, 7, nil] }
+      expected_merge = {
+        'first_updated_at' => old_time,
+        'last_updated_at' => current_time,
+        'file_hash' => 'abcd',
+        'data' => [10, 14, nil]
+      }
+      new_data = {
+        'first_updated_at' => current_time,
+        'last_updated_at' => current_time,
+        'file_hash' => 'abcd',
+        'data' => [5, 7, nil]
+      }
+      expected_result = {
+        '/projects/coverband_demo/app/controllers/application_controller.rb' => new_data,
+        '/projects/coverband_demo/config/coverband.rb' => expected_merge,
+        '/projects/coverband_demo/config/initializers/assets.rb' => old_data,
+        '/projects/coverband_demo/config/initializers/cookies_serializer.rb' => old_data,
+        '/projects/coverband_demo/config/initializers/filter_logging.rb' => new_data,
+        '/projects/coverband_demo/config/initializers/wrap_parameters.rb' => new_data
+      }
+      assert_equal expected_result, @store.send(:merge_reports, new_report, old_report)
+    end
   end
 end

data/test/coverband/adapters/file_store_test.rb

@@ -3,45 +3,56 @@
 require File.expand_path('../../test_helper', File.dirname(__FILE__))
 
 class AdaptersFileStoreTest < Minitest::Test
-  def setup
-    super
-    @test_file_path = '/tmp/coverband_filestore_test_path.json'
-    File.open(@test_file_path, 'w') { |f| f.write(test_data.to_json) }
-    @store = Coverband::Adapters::FileStore.new(@test_file_path)
-  end
-
-  def test_covered_lines_for_file
-    assert_equal @store.covered_lines_for_file('dog.rb')[0],  1
-    assert_equal @store.covered_lines_for_file('dog.rb')[1],  2
-  end
-
-  def test_covered_lines_when_null
-    assert_equal @store.covered_lines_for_file('none.rb'), []
-  end
 
-  def test_covered_files
-    assert_equal @store.covered_files, ['dog.rb']
+  def test_covered_lines_when_no_file
+    @store = Coverband::Adapters::FileStore.new('')
+    expected = {}
+    assert_equal expected, @store.coverage
   end
 
-  def test_clear
-    @store.clear!
-    assert_equal false, File.exist?(@test_file_path)
-  end
-
-  def test_save_report
-    mock_file_hash
-    @store.send(:save_report, 'cat.rb' => [0, 1])
-    assert_equal @store.covered_lines_for_file('cat.rb')[1], 1
-  end
-
-  private
-
-  def test_data
-    {
-      'dog.rb' => { 'data' => [1, 2, nil],
-                    'file_hash' => 'abcd',
-                    'first_updated_at' => 1541968729,
-                    'last_updated_at' => 1541968729 }
-    }
+  describe 'Coverband::Adapters::FileStore with file' do
+    def setup
+      super
+      @test_file_path = '/tmp/coverband_filestore_test_path.json'
+      File.open(@test_file_path, 'w') { |f| f.write(test_data.to_json) }
+      @store = Coverband::Adapters::FileStore.new(@test_file_path)
+    end
+
+    def test_size
+      assert @store.size > 1
+    end
+
+    def test_coverage
+      assert_equal @store.coverage['dog.rb']['data'][0],  1
+      assert_equal @store.coverage['dog.rb']['data'][1],  2
+    end
+
+    def test_covered_lines_when_null
+      assert_nil @store.coverage['none.rb']
+    end
+
+    def test_covered_files
+      assert_equal @store.covered_files, ['dog.rb']
+    end
+
+    def test_clear
+      @store.clear!
+      assert_equal false, File.exist?(@test_file_path)
+    end
+
+    def test_save_report
+      mock_file_hash
+      @store.send(:save_report, 'cat.rb' => [0, 1])
+      assert_equal @store.coverage['cat.rb']['data'][1], 1
+    end
+
+    def test_data
+      {
+        'dog.rb' => { 'data' => [1, 2, nil],
+                      'file_hash' => 'abcd',
+                      'first_updated_at' => 1541968729,
+                      'last_updated_at' => 1541968729 }
+      }
+    end
   end
 end