coverband 2.0.0.alpha1 → 2.0.0

data/Rakefile

@@ -14,3 +14,8 @@ Rake::TestTask.new(:test) do |test|
   test.test_files = FileList['test/unit/*_test.rb']
   test.verbose = true
 end
+
+desc 'load irb with this gem'
+task :console do
+  exec 'irb -I lib -r coverband'
+end

data/changes.md

@@ -1,6 +1,37 @@
+# Future Roadmap
+
+### Coverband 3.0
+
+Will be the fully modern release that drops maintenance legacy support in favor of increased performance, ease of use, and maintainability.
+
+* expects to drop Tracepoint collection engine
+* expects to drop anything below Ruby 2.3
+* Release will be aimed as significantly simplifying ease of use
+   * expects to drop the concept of baseline recordings
+   * improve support for eager-loading
+   * add built-in support for easy loading via Railties
+   * expects to add safe list support to force reload files one wants coverage on that may happen outside of the standard load order
+   * built in support for activejob, sidekiq, and other common frameworks
+
+# Released
+
+### 2.0.0
+
+Major release with various backwards compatibility breaking changes (generally related to the configuration). The 2.0 lifecycle will act as a mostly easy upgrade that supports past users looking to move to the much faster new Coverage Adapter.
+
+* Continues to support Ruby 2.0 and up
+* supports multiple collect engines, introducing the concept of multiple collector adapters
+* extends the concepts of multiple storage adapters, enabling additional authors to help support Kafka, graphite, other adapters
+* old require based loading, but working towards deprecating the entire baseline concept
+* Introduces massive performance enhancements by moving to Ruby `Coverage` based collection
+   * Opposed to sampling this is now a reporting frequency, when using `Coverage` collector
+* Reduced configuration complexity
+* Refactoring the code preparing for more varied storage and reporting options
+* Drop Redis as a gem runtime_dependency
+
 ### 1.5.0
 
-This is a major release with significant refactoring a stepping stone for a 2.0 release.
+This is a significant release with significant refactoring a stepping stone for a 2.0 release.
 
 * staging a changes.md document!
 * refactored out full abstraction for stores
@@ -20,4 +51,4 @@ This is a major release with significant refactoring a stepping stone for a 2.0
 
 * This was a small fix release addressing some issues
 * mostly readme updates
-* last release prior to having a changes document!
+* last release prior to having a changes document!

data/coverband.gemspec

@@ -28,11 +28,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'sinatra'
   spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'redis'
+  spec.add_development_dependency 'benchmark-ips'
   # add when debugging
   # require 'byebug'; byebug
-  spec.add_development_dependency 'byebug'
+  #spec.add_development_dependency 'byebug'
+  # deprecate when dropping support for older ruby
   spec.add_runtime_dependency 'json'
+  # todo make an optional dependency for simplecov reports
+  # also likely should just require simplecov-html not the whole lib
   spec.add_runtime_dependency 'simplecov', '> 0.11.1'
-  # TODO: make redis optional dependancy as we add additional adapters
-  spec.add_runtime_dependency 'redis'
 end

data/lib/coverband.rb

@@ -2,11 +2,10 @@
 
 require 'logger'
 require 'json'
-# TODO: move to only be request if using redis store
-require 'redis'
 
 require 'coverband/version'
 require 'coverband/configuration'
+require 'coverband/adapters/base'
 require 'coverband/adapters/redis_store'
 require 'coverband/adapters/memory_cache_store'
 require 'coverband/adapters/file_store'

data/lib/coverband/adapters/base.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Coverband
+  module Adapters
+    class Base
+      def initialize
+        raise 'abstract'
+      end
+
+      def clear!
+        raise 'abstract'
+      end
+
+      def save_report(report)
+        raise 'abstract'
+      end
+
+      def coverage
+        raise 'abstract'
+      end
+
+      def covered_files
+        raise 'abstract'
+      end
+
+      def covered_lines_for_file(file)
+        raise 'abstract'
+      end
+    end
+  end
+end

data/lib/coverband/adapters/file_store.rb

@@ -2,7 +2,7 @@
 
 module Coverband
   module Adapters
-    class FileStore
+    class FileStore < Base
       attr_accessor :path
 
       def initialize(path, _opts = {})

data/lib/coverband/adapters/memory_cache_store.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+###
+# TODO current benchmarks aren't showing much advantage from this wrapped cache approach
+# re-evaluate before 2.0.0 release 
+###
 module Coverband
   module Adapters
-    class MemoryCacheStore
+    class MemoryCacheStore < Base
       attr_accessor :store
 
       def initialize(store)

data/lib/coverband/adapters/redis_store.rb

@@ -2,8 +2,8 @@
 
 module Coverband
   module Adapters
-    class RedisStore
-      BASE_KEY = 'coverband1'
+    class RedisStore < Base
+      BASE_KEY = 'coverband2'
 
       def initialize(redis, opts = {})
         @redis = redis

data/lib/coverband/collectors/base.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+####
+# TODO refactor this along with the coverage and trace collector
+####
 module Coverband
   module Collectors
     class Base
@@ -20,7 +23,7 @@ module Coverband
 
       def stop
         @enabled = false
-        unset_tracer
+        stop_coverage
       end
 
       def sample
@@ -49,7 +52,6 @@ module Coverband
         @sample_percentage = Coverband.configuration.percentage
         @store = Coverband.configuration.store
         @store = Coverband::Adapters::MemoryCacheStore.new(@store) if Coverband.configuration.memory_caching
-        @stats    = Coverband.configuration.stats
         @verbose  = Coverband.configuration.verbose
         @logger   = Coverband.configuration.logger
         @current_thread = Thread.current
@@ -67,19 +69,11 @@ module Coverband
       end
 
       def record_coverage
-        if @enabled && !failed_recently?
-          set_tracer
-        else
-          unset_tracer
-        end
-        # support old ruby before &. safe digging
-        @stats.increment "coverband.request.recorded.#{@enabled}" if @stats
-      rescue RuntimeError => err
-        failed!
-        if @verbose
-          @logger.info 'error stating recording coverage'
-          @logger.info "error: #{err.inspect} #{err.message}"
-        end
+        raise 'abstract'
+      end
+
+      def stop_coverage
+        raise 'abstract'
       end
 
       def report_coverage
@@ -92,14 +86,6 @@ module Coverband
         @ignore_patterns.none? { |pattern| file.include?(pattern) } && file.start_with?(@project_directory)
       end
 
-      def set_tracer
-        raise 'abstract'
-      end
-
-      def unset_tracer
-        raise 'abstract'
-      end
-
       def output_file_line_usage
         @logger.info 'coverband debug coverband file:line usage:'
         @file_line_usage.sort_by { |_key, value| value.length }.each do |pair|

data/lib/coverband/collectors/coverage.rb

@@ -7,6 +7,10 @@ module Coverband
         # noop
       end
 
+      def stop_coverage
+        # noop
+      end
+
       def report_coverage
         unless @enabled
           @logger.info 'coverage disabled' if @verbose
@@ -32,16 +36,7 @@ module Coverband
         end
 
         if @store
-          if @stats
-            @before_time = Time.now
-            @stats.count 'coverband.files.recorded_files', @file_line_usage.length
-          end
-
           @store.save_report(@file_line_usage)
-          if @stats
-            @time_spent = Time.now - @before_time
-            @stats.timing 'coverband.files.recorded_time', @time_spent
-          end
           @file_line_usage.clear
         elsif @verbose
           @logger.info 'coverage report: '
@@ -59,16 +54,6 @@ module Coverband
         end
       end
 
-      protected
-
-      def set_tracer
-        # no op
-      end
-
-      def unset_tracer
-        # no op
-      end
-
       private
 
       def array_diff(latest, original)
@@ -97,22 +82,6 @@ module Coverband
           new_results = current_coverage
         end
 
-        # if current_coverage['/Users/danmayer/projects/coverage_rails_benchmark/app/controllers/posts_controller.rb']
-        #   puts "total"
-        #   puts current_coverage['/Users/danmayer/projects/coverage_rails_benchmark/app/controllers/posts_controller.rb'].inspect
-        # end
-        #
-        #
-        # if previous_results && previous_results['/Users/danmayer/projects/coverage_rails_benchmark/app/controllers/posts_controller.rb']
-        #   puts "prev"
-        #   puts previous_results['/Users/danmayer/projects/coverage_rails_benchmark/app/controllers/posts_controller.rb'].inspect
-        # end
-        #
-        # if new_results['/Users/danmayer/projects/coverage_rails_benchmark/app/controllers/posts_controller.rb']
-        #   puts "new"
-        #   puts new_results['/Users/danmayer/projects/coverage_rails_benchmark/app/controllers/posts_controller.rb'].inspect
-        # end
-
         add_previous_results(current_coverage)
         new_results.dup
       end

data/lib/coverband/collectors/trace.rb

@@ -7,10 +7,29 @@ module Coverband
       def reset_instance
         super
         @tracer_set = false
+        @trace_point_events = [:line]
         @trace = create_trace_point
         self
       end
 
+      def record_coverage
+        if @enabled && !failed_recently?
+          set_tracer
+        else
+          unset_tracer
+        end
+      rescue RuntimeError => err
+        failed!
+        if @verbose
+          @logger.info 'error stating recording coverage'
+          @logger.info "error: #{err.inspect} #{err.message}"
+        end
+      end
+
+      def stop_coverage
+        unset_tracer
+      end
+
       def report_coverage
         unless @enabled
           @logger.info 'coverage disabled' if @verbose
@@ -30,15 +49,7 @@ module Coverband
         end
 
         if @store
-          if @stats
-            @before_time = Time.now
-            @stats.count 'coverband.files.recorded_files', @file_line_usage.length
-          end
           @store.save_report(@file_line_usage)
-          if @stats
-            @time_spent = Time.now - @before_time
-            @stats.timing 'coverband.files.recorded_time', @time_spent
-          end
           @file_line_usage.clear
         elsif @verbose
           @logger.info 'coverage report: '
@@ -82,9 +93,10 @@ module Coverband
       end
 
       def create_trace_point
-        TracePoint.new(*Coverband.configuration.trace_point_events) do |tp|
+        TracePoint.new(*@trace_point_events) do |tp|
           if Thread.current == @current_thread
             file = tp.path
+
             unless @ignored_files.include?(file)
               if track_file?(file)
                 add_file(file, tp.lineno)

data/lib/coverband/configuration.rb

@@ -3,16 +3,15 @@
 module Coverband
   class Configuration
     attr_accessor :redis, :root_paths, :root,
-                  :ignore, :additional_files, :percentage, :verbose, :reporter,
-                  :stats, :startup_delay, :trace_point_events,
-                  :include_gems, :memory_caching, :s3_bucket, :coverage_file,
+                  :ignore, :additional_files, :percentage, :verbose,
+                  :reporter, :startup_delay, :memory_caching,
+                  :include_gems, :s3_bucket,
                   :collector, :disable_on_failure_for
-    attr_writer :logger, :store
+    attr_writer :logger
 
     def initialize
       @root = Dir.pwd
       @redis = nil
-      @stats = nil
       @root_paths = []
       @ignore = []
       @additional_files = []
@@ -20,12 +19,14 @@ module Coverband
       @percentage = 0.0
       @verbose = false
       @reporter = 'scov'
-      @collector = 'trace'
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3.0')
+        @collector = 'trace'
+      else
+        @collector = 'coverage'
+      end
       @logger = Logger.new(STDOUT)
       @startup_delay = 0
-      @trace_point_events = [:line]
       @memory_caching = false
-      @coverage_file = nil
       @store = nil
       @disable_on_failure_for = nil
     end
@@ -34,15 +35,19 @@ module Coverband
       @logger ||= Logger.new(STDOUT)
     end
 
-    # TODO: considering removing @redis / @coveragefile and have user set store directly
     def store
       return @store if @store
-      if redis
+      raise 'no valid store configured'
+    end
+
+    def store=(store)
+      if store.is_a?(Coverband::Adapters::Base)
+        @store = store
+      elsif defined?(Redis) && store.is_a?(Redis)
         @store = Coverband::Adapters::RedisStore.new(redis)
-      elsif Coverband.configuration.coverage_file
+      elsif store.is_a?(String)
         @store = Coverband::Adapters::FileStore.new(coverage_file)
       end
-      @store
     end
   end
 end

data/lib/coverband/s3_web.rb

@@ -4,7 +4,7 @@ require 'sinatra/base'
 
 module Coverband
   class S3Web < Sinatra::Base
-    set :public_folder, proc { File.expand_path('public', Gem::Specification.find_by_name('simplecov-html').full_gem_path) }
+    set :public_folder, proc(){ File.expand_path('public', Gem::Specification.find_by_name('simplecov-html').full_gem_path) }
 
     get '/' do
       s3.get_object(bucket: Coverband.configuration.s3_bucket, key: 'coverband/index.html').body.read
@@ -13,6 +13,12 @@ module Coverband
     private
 
     def s3
+      begin
+        require 'aws-sdk'
+      rescue StandardError
+        Coverband.configuration.logger.error "coverband requires 'aws-sdk' in order use S3ReportWriter."
+        return
+      end
       @s3 ||= Aws::S3::Client.new
     end
   end

data/lib/coverband/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coverband
-  VERSION = '2.0.0.alpha1'
+  VERSION = '2.0.0'
 end

data/test/benchmarks/benchmark.rake

@@ -1,10 +1,34 @@
 # frozen_string_literal: true
+namespace :benchmarks do
 
-require 'coverband'
-require 'redis'
-require File.join(File.dirname(__FILE__), 'dog')
+  # https://github.com/evanphx/benchmark-ips
+  # Enable and start GC before each job run. Disable GC afterwards.
+  #
+  # Inspired by https://www.omniref.com/ruby/2.2.1/symbols/Benchmark/bm?#annotation=4095926&line=182
+  class GCSuite
+    def warming(*)
+      run_gc
+    end
+
+    def running(*)
+      run_gc
+    end
+
+    def warmup_stats(*)
+    end
+
+    def add_report(*)
+    end
+
+    private
+
+    def run_gc
+      GC.enable
+      GC.start
+      GC.disable
+    end
+  end
 
-namespace :benchmarks do
   def classifier_dir
     File.join(File.dirname(__FILE__), 'classifier-reborn')
   end
@@ -17,59 +41,70 @@ namespace :benchmarks do
     # rubocop:enable Style/IfUnlessModifier
   end
 
-  desc 'set up coverband default redis'
+  desc 'setup standard benchmark'
   task :setup do
     clone_classifier
     $LOAD_PATH.unshift(File.join(classifier_dir, 'lib'))
     require 'benchmark'
+    require 'benchmark/ips'
+
+    # TODO ok this is interesting and weird
+    # basically the earlier I require coverage and
+    # then require files the larger perf impact
+    # this is somewhat expected but can lead to significant perf diffs
+    # for example moving `require 'classifier-reborn'` below the coverage.start
+    # results in 1.5x slower vs "difference falls within error"
+    # moving from 5 second of time to 12 still shows slower based on when classifier is required
+    # make sure to be plugged in while benchmarking ;) Otherwise you get very unreliable results
     require 'classifier-reborn'
+    if ENV['COVERAGE']
+      puts 'Coverage library loaded and started'
+      require 'coverage'
+      ::Coverage.start
+    end
+    require 'redis'
+    require 'coverband'
+    require File.join(File.dirname(__FILE__), 'dog')
+  end
 
+  desc 'set up coverband tracepoint Redis'
+  task :setup_redis do
     Coverband.configure do |config|
       config.redis              = Redis.new
       config.root               = Dir.pwd
-      config.startup_delay      = 0
       config.percentage         = 100.0
       config.logger             = $stdout
-      config.verbose            = false
-      # config.memory_caching     = true
-      # config.trace_point_events = [:call]
+      config.collector          = 'trace'
+      config.memory_caching     = ENV['MEMORY_CACHE'] ? true : false
+      config.store              = Coverband::Adapters::RedisStore.new(Redis.new)
     end
   end
 
-  desc 'set up coverband with coverage redis'
-  task :setup_coverage do
-    clone_classifier
-    $LOAD_PATH.unshift(File.join(classifier_dir, 'lib'))
-    require 'benchmark'
-    require 'classifier-reborn'
-
+  desc 'set up coverband tracepoint filestore'
+  task :setup_file do
     Coverband.configure do |config|
-      config.redis              = Redis.new
       config.root               = Dir.pwd
-      config.startup_delay      = 0
       config.percentage         = 100.0
       config.logger             = $stdout
-      config.verbose            = false
-      config.collector          = 'coverage'
+      config.collector          = 'trace'
+      config.memory_caching     = ENV['MEMORY_CACHE'] ? true : false
+      config.store              = Coverband::Adapters::FileStore.new('/tmp/benchmark_store.json')
     end
   end
 
-  desc 'set up coverband filestore'
-  task :setup_file do
-    clone_classifier
-    $LOAD_PATH.unshift(File.join(classifier_dir, 'lib'))
-    require 'benchmark'
-    require 'classifier-reborn'
-
+  ###
+  # This benchmark always needs to be run last
+  # as requiring coverage changes how Ruby interprets the code
+  ###
+  desc 'set up coverband with coverage Redis'
+  task :setup_coverage do
     Coverband.configure do |config|
-      config.redis              = nil
-      config.store              = nil
       config.root               = Dir.pwd
-      config.startup_delay      = 0
       config.percentage         = 100.0
       config.logger             = $stdout
-      config.verbose            = false
-      config.coverage_file = '/tmp/benchmark_store.json'
+      config.collector          = 'coverage'
+      config.memory_caching     = ENV['MEMORY_CACHE'] ? true : false
+      config.store              = Coverband::Adapters::RedisStore.new(Redis.new)
     end
   end
 
@@ -103,49 +138,52 @@ namespace :benchmarks do
     10_000.times { Dog.new.bark }
   end
 
-  def run_work
-    puts "benchmark for: #{Coverband.configuration.inspect}"
-    puts "store: #{Coverband.configuration.store.inspect}"
-    Benchmark.bm(15) do |x|
+  def run_work(hold_work = false)
+    suite = GCSuite.new
+    #puts "benchmark for: #{Coverband.configuration.inspect}"
+    #puts "store: #{Coverband.configuration.store.inspect}"
+    Benchmark.ips do |x|
+      x.config(:time => 12, :warmup => 5, :suite => suite)
       x.report 'coverband' do
-        SAMPLINGS.times do
-          Coverband::Collectors::Base.instance.sample do
-            work
-          end
+        Coverband::Collectors::Base.instance.sample do
+          work
         end
       end
-
+      Coverband::Collectors::Base.instance.stop
       x.report 'no coverband' do
-        SAMPLINGS.times do
-          work
-        end
+        work
       end
+      x.hold! 'temp_results' if hold_work
+      x.compare!
     end
-    Coverband::Collectors::Base.instance.stop
     Coverband::Collectors::Base.instance.reset_instance
   end
 
   desc 'runs benchmarks on default redis setup'
-  task run: :setup do
-    puts 'Coverband tracepoint configured with default redis store'
-    SAMPLINGS = 5
+  task run_redis: [:setup, :setup_redis] do
+    puts 'Coverband tracepoint configured with default Redis store'
     run_work
   end
 
   desc 'runs benchmarks file store'
-  task run_file: :setup_file do
+  task run_file: [:setup, :setup_file] do
     puts 'Coverband tracepoint configured with file store'
-    SAMPLINGS = 5
     run_work
   end
 
   desc 'runs benchmarks coverage'
-  task run_coverage: :setup_coverage do
-    puts 'Coverband Coverage configured with to use default redis store'
-    SAMPLINGS = 5
-    run_work
+  task run_coverage: [:setup, :setup_coverage] do
+    puts 'Coverband Coverage configured with to use default Redis store'
+    run_work(true)
+  end
+
+  desc 'compare Coverband Ruby Coverage with normal Ruby'
+  task :compare_coverage do
+    puts 'comparing with Coverage loaded and not, this takes some time for output...'
+    puts `COVERAGE=true rake benchmarks:run_coverage`
+    puts `rake benchmarks:run_coverage`
   end
 end
 
 desc 'runs benchmarks'
-task benchmarks: ['benchmarks:run_file', 'benchmarks:run', 'benchmarks:run_coverage']
+task benchmarks: ['benchmarks:run_file', 'benchmarks:run_redis', 'benchmarks:compare_coverage']