rspec-tracer 0.7.0 → 0.9.2

data/lib/rspec_tracer/remote_cache/repo.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  module RemoteCache
+    class Repo
+      class RepoError < StandardError; end
+
+      attr_reader :branch_name, :branch_ref, :branch_refs, :ancestry_refs, :cache_refs
+
+      def initialize(aws)
+        @aws = aws
+        @branch_name = ENV['GIT_BRANCH'].chomp
+
+        raise RepoError, 'GIT_BRANCH environment variable is not set' if @branch_name.nil?
+
+        fetch_head_ref
+        fetch_branch_ref
+        fetch_ancestry_refs
+        fetch_branch_refs
+        generate_cache_refs
+      end
+
+      private
+
+      def fetch_head_ref
+        @head_ref = `git rev-parse HEAD`.chomp
+
+        raise RepoError, 'Could not find HEAD commit sha' unless $CHILD_STATUS.success?
+      end
+
+      def fetch_branch_ref
+        @merged_parents = []
+        @ignored_refs = []
+
+        unless merged?
+          @branch_ref = @head_ref
+
+          return
+        end
+
+        @ignored_refs << @head_ref
+
+        fetch_merged_parents
+        fetch_merged_branch_ref
+      end
+
+      def fetch_ancestry_refs
+        ref_list = `git rev-list --max-count=25 #{@branch_ref}`.chomp.split
+
+        raise RepoError, 'Could not find ancestry refs' unless $CHILD_STATUS.success?
+
+        ref_list = ref_list.to_set - @ignored_refs
+        @ancestry_refs = refs_committer_timestamp(ref_list.to_a)
+
+        return if @ancestry_refs.empty?
+
+        print_refs(@ancestry_refs, 'ancestry')
+      end
+
+      def fetch_branch_refs
+        unless @aws.branch_refs?(@branch_name)
+          puts "No branch refs for #{@branch_name} branch found in S3"
+
+          @branch_refs = {}
+
+          return
+        end
+
+        download_branch_refs
+      end
+
+      def generate_cache_refs
+        ref_list = @ancestry_refs.merge(@branch_refs)
+
+        if ref_list.empty?
+          @cache_refs = {}
+
+          return
+        end
+
+        @cache_refs = ref_list.sort_by { |_, timestamp| -timestamp }.to_h
+
+        print_refs(@cache_refs, 'cache')
+      end
+
+      def merged?
+        system('git', 'rev-parse', 'HEAD^2', out: File::NULL, err: File::NULL)
+      end
+
+      def fetch_merged_parents
+        first_parent = `git rev-parse HEAD^1`.chomp
+        @merged_parents << first_parent if $CHILD_STATUS.success?
+
+        second_parent = `git rev-parse HEAD^2`.chomp
+        @merged_parents << second_parent if $CHILD_STATUS.success?
+
+        raise RepoError, 'Could not find merged commit parents' if @merged_parents.length != 2
+      end
+
+      def fetch_merged_branch_ref
+        @origin_head_ref = `git rev-parse origin/HEAD`.chomp
+        @branch_ref = nil
+
+        if @merged_parents.first != @origin_head_ref
+          @branch_ref = @head_ref
+          @ignored_refs = []
+
+          return
+        end
+
+        @branch_ref = @merged_parents.last
+        @ignored_refs = @ignored_refs.to_set | `git rev-list #{@branch_ref}..origin/HEAD`.chomp.split
+
+        raise RepoError, 'Could not find ignored refs' unless $CHILD_STATUS.success?
+      end
+
+      def refs_committer_timestamp(ref_list)
+        return {} if ref_list.empty?
+
+        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
+          git show
+            --no-patch
+            --format="%H %ct"
+            #{ref_list.join(' ')}
+        COMMAND
+
+        ref_list = `#{command}`.chomp
+
+        raise RepoError, 'Could not find ancestry refs' unless $CHILD_STATUS.success?
+
+        ref_list.split("\n").map(&:split).to_h.transform_values(&:to_i)
+      end
+
+      def download_branch_refs
+        file_name = File.join(RSpecTracer.cache_path, 'branch_refs.json')
+
+        if @aws.download_branch_refs(branch_name, file_name)
+          @branch_refs = JSON.parse(File.read(file_name)).transform_values(&:to_i)
+
+          return if @branch_refs.empty?
+
+          filter_branch_refs
+          print_refs(@branch_refs, 'branch')
+        else
+          @branch_refs = {}
+
+          File.rm_f(file_name)
+
+          puts "Failed to fetch branch refs for #{@branch_name} branch"
+        end
+      end
+
+      def filter_branch_refs
+        if @ancestry_refs.empty?
+          @branch_refs = @branch_refs.sort_by { |_, timestamp| -timestamp }.first(25).to_h
+
+          return
+        end
+
+        oldest_ancestry_time = @ancestry_refs.values.min
+
+        @branch_refs = @branch_refs
+          .select { |_, timestamp| timestamp >= oldest_ancestry_time }
+          .sort_by { |_, timestamp| -timestamp }
+          .first(25)
+          .to_h
+      end
+
+      def print_refs(refs, type)
+        puts "Fetched the following #{type} refs for #{@branch_name} branch:"
+        puts refs.map { |ref, timestamp| "  * #{ref} (commit timestamp: #{timestamp})" }.join("\n")
+      end
+    end
+  end
+end

data/lib/rspec_tracer/remote_cache/validator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  module RemoteCache
+    class Validator
+      CACHE_FILES_PER_TEST_SUITE = 8
+
+      def initialize
+        @test_suite_id = ENV['TEST_SUITE_ID']
+        @test_suites = ENV['TEST_SUITES']
+
+        if @test_suite_id.nil? ^ @test_suites.nil?
+          raise(
+            ValidationError,
+            'Both the enviornment variables TEST_SUITE_ID and TEST_SUITES are not set'
+          )
+        end
+
+        setup
+      end
+
+      def valid?(ref, cache_files)
+        last_run_regex = Regexp.new(format(@last_run_files_regex, ref: ref))
+
+        return false if cache_files.count { |file| file.match?(last_run_regex) } != @last_run_files_count
+
+        cache_regex = Regexp.new(format(@cached_files_regex, ref: ref))
+
+        cache_files.count { |file| file.match?(cache_regex) } == @cached_files_count
+      end
+
+      private
+
+      def setup
+        if @test_suites.nil?
+          @last_run_files_count = 1
+          @last_run_files_regex = '/%<ref>s/last_run.json$'
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE
+          @cached_files_regex = '/%<ref>s/[0-9a-f]{32}/.+.json'
+        else
+          @test_suites = @test_suites.to_i
+          @test_suites_regex = (1..@test_suites).to_a.join('|')
+
+          @last_run_files_count = @test_suites
+          @last_run_files_regex = "/%<ref>s/(#{@test_suites_regex})/last_run.json$"
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE * @test_suites
+          @cached_files_regex = "/%<ref>s/(#{@test_suites_regex})/[0-9a-f]{32}/.+.json$"
+        end
+      end
+    end
+  end
+end

data/lib/rspec_tracer/reporter.rb

@@ -2,9 +2,10 @@
 
 module RSpecTracer
   class Reporter
-    attr_reader :all_examples, :possibly_flaky_examples, :flaky_examples, :pending_examples,
-                :all_files, :modified_files, :deleted_files, :dependency, :reverse_dependency,
-                :examples_coverage, :last_run
+    attr_reader :all_examples, :interrupted_examples, :possibly_flaky_examples,
+                :flaky_examples, :pending_examples, :all_files, :modified_files,
+                :deleted_files, :dependency, :reverse_dependency, :examples_coverage,
+                :last_run
 
     def initialize
       initialize_examples
@@ -15,6 +16,7 @@ module RSpecTracer
 
     def register_example(example)
       @all_examples[example[:example_id]] = example
+      @duplicate_examples[example[:example_id]] << example
     end
 
     def on_example_skipped(example_id)
@@ -36,8 +38,21 @@ module RSpecTracer
       @all_examples[example_id][:execution_result] = formatted_execution_result(result)
     end
 
+    def register_interrupted_examples
+      @all_examples.each_pair do |example_id, example|
+        next if example.key?(:execution_result)
+
+        @interrupted_examples << example_id
+      end
+
+      return if @interrupted_examples.empty?
+
+      puts "RSpec tracer is not processing #{@interrupted_examples.count} interrupted examples"
+    end
+
     def register_deleted_examples(seen_examples)
       @deleted_examples = seen_examples.keys.to_set - (@skipped_examples | @all_examples.keys)
+      @deleted_examples -= @interrupted_examples
 
       @deleted_examples.select! do |example_id|
         example = seen_examples[example_id]
@@ -62,6 +77,10 @@ module RSpecTracer
       @pending_examples << example_id
     end
 
+    def example_interrupted?(example_id)
+      @interrupted_examples.include?(example_id)
+    end
+
     def example_passed?(example_id)
       @passed_examples.include?(example_id)
     end
@@ -106,6 +125,17 @@ module RSpecTracer
       file_deleted?(file_name) || file_modified?(file_name)
     end
 
+    def incorrect_analysis?
+      @duplicate_examples.select! { |_, examples| examples.count > 1 }
+
+      return false if @duplicate_examples.empty?
+
+      print_not_use_notice
+      print_duplicate_examples
+
+      true
+    end
+
     def register_dependency(example_id, file_name)
       @dependency[example_id] << file_name
     end
@@ -116,6 +146,8 @@ module RSpecTracer
 
     def generate_reverse_dependency_report
       @dependency.each_pair do |example_id, files|
+        next if @interrupted_examples.include?(example_id)
+
         example_file = @all_examples[example_id][:rerun_file_name]
 
         files.each do |file_name|
@@ -133,9 +165,11 @@ module RSpecTracer
         pid: RSpecTracer.pid,
         actual_count: RSpec.world.example_count + @skipped_examples.count,
         example_count: RSpec.world.example_count,
+        interrupted_examples: @interrupted_examples.count,
         failed_examples: @failed_examples.count,
         skipped_examples: @skipped_examples.count,
-        pending_examples: @pending_examples.count
+        pending_examples: @pending_examples.count,
+        flaky_examples: @flaky_examples.count
       }
     end
 
@@ -169,6 +203,8 @@ module RSpecTracer
 
     def initialize_examples
       @all_examples = {}
+      @duplicate_examples = Hash.new { |examples, example_id| examples[example_id] = [] }
+      @interrupted_examples = Set.new
       @passed_examples = Set.new
       @possibly_flaky_examples = Set.new
       @flaky_examples = Set.new
@@ -226,6 +262,45 @@ module RSpecTracer
       @reverse_dependency = report.to_h
     end
 
+    def print_not_use_notice
+      justify = ' ' * 4
+      four_justify = justify * 4
+
+      puts '=' * 80
+      puts "#{four_justify}IMPORTANT NOTICE -- DO NOT USE RSPEC TRACER"
+      puts '=' * 80
+      puts "#{justify}It would be best to make changes so that the RSpec tracer can uniquely"
+      puts "#{justify}identify all the examples, and then you can enable the RSpec tracer back."
+      puts '=' * 80
+      puts
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def print_duplicate_examples
+      total = @duplicate_examples.sum { |_, examples| examples.length }
+
+      puts "RSpec tracer could not uniquely identify the following #{total} examples:"
+
+      justify = ' ' * 2
+      nested_justify = justify * 3
+
+      @duplicate_examples.each_pair do |example_id, examples|
+        puts "#{justify}- Example ID: #{example_id} (#{examples.count} examples)"
+
+        examples.each do |example|
+          description = example[:full_description].strip
+          file_name = example[:rerun_file_name].sub(%r{^/}, '')
+          line_number = example[:rerun_line_number]
+          location = "#{file_name}:#{line_number}"
+
+          puts "#{nested_justify}* #{description} (#{location})"
+        end
+      end
+
+      puts
+    end
+    # rubocop:enable Metrics/AbcSize
+
     def write_all_examples_report
       file_name = File.join(@cache_dir, 'all_examples.json')
 

data/lib/rspec_tracer/rspec_runner.rb

@@ -3,13 +3,23 @@
 module RSpecTracer
   module RSpecRunner
     # rubocop:disable Metrics/AbcSize
-    def run_specs(_example_groups)
+    def run_specs(example_groups)
       actual_count = RSpec.world.example_count
+      RSpecTracer.no_examples = actual_count.zero?
+
+      if RSpecTracer.no_examples
+        RSpecTracer.running = true
+
+        super(example_groups)
+
+        return
+      end
+
       starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      filtered_examples, example_groups = RSpecTracer.filter_examples
+      filtered_examples, filtered_example_groups = RSpecTracer.filter_examples
 
       RSpec.world.instance_variable_set(:@filtered_examples, filtered_examples)
-      RSpec.world.instance_variable_set(:@example_groups, example_groups)
+      RSpec.world.instance_variable_set(:@example_groups, filtered_example_groups)
 
       current_count = RSpec.world.example_count
       ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -23,7 +33,7 @@ module RSpecTracer
 
       RSpecTracer.running = true
 
-      super(example_groups)
+      super(filtered_example_groups)
     end
     # rubocop:enable Metrics/AbcSize
   end

data/lib/rspec_tracer/runner.rb

@@ -8,6 +8,7 @@ module RSpecTracer
     EXAMPLE_RUN_REASON = {
       explicit_run: 'Explicit run',
       no_cache: 'No cache',
+      interrupted: 'Interrupted previously',
       flaky_example: 'Flaky example',
       failed_example: 'Failed previously',
       pending_example: 'Pending previously',
@@ -59,10 +60,18 @@ module RSpecTracer
       @reporter.on_example_pending(example_id, execution_result)
     end
 
+    def register_interrupted_examples
+      @reporter.register_interrupted_examples
+    end
+
     def register_deleted_examples
       @reporter.register_deleted_examples(@cache.all_examples)
     end
 
+    def incorrect_analysis?
+      @reporter.incorrect_analysis?
+    end
+
     # rubocop:disable Metrics/AbcSize
     def generate_missed_coverage
       missed_coverage = Hash.new do |files_coverage, file_path|
@@ -73,6 +82,7 @@ module RSpecTracer
 
       @cache.cached_examples_coverage.each_pair do |example_id, example_coverage|
         example_coverage.each_pair do |file_path, line_coverage|
+          next if @reporter.example_interrupted?(example_id)
           next unless @reporter.example_skipped?(example_id)
 
           file_name = RSpecTracer::SourceFile.file_name(file_path)
@@ -93,6 +103,8 @@ module RSpecTracer
       filtered_files = Set.new
 
       examples_coverage.each_pair do |example_id, example_coverage|
+        next if @reporter.example_interrupted?(example_id)
+
         register_example_files_dependency(example_id)
 
         example_coverage.each_key do |file_path|
@@ -118,6 +130,8 @@ module RSpecTracer
         @reporter.register_source_file(source_file)
 
         @reporter.all_examples.each_key do |example_id|
+          next if @reporter.example_interrupted?(example_id)
+
           @reporter.register_dependency(example_id, source_file[:file_name])
         end
       end
@@ -165,6 +179,7 @@ module RSpecTracer
     end
 
     def filter_by_example_status
+      add_previously_interrupted_examples
       add_previously_flaky_examples
       add_previously_failed_examples
       add_previously_pending_examples
@@ -179,10 +194,17 @@ module RSpecTracer
       end
     end
 
+    def add_previously_interrupted_examples
+      @cache.interrupted_examples.each do |example_id|
+        @filtered_examples[example_id] = EXAMPLE_RUN_REASON[:interrupted]
+      end
+    end
+
     def add_previously_flaky_examples
       @cache.flaky_examples.each do |example_id|
         @filtered_examples[example_id] = EXAMPLE_RUN_REASON[:flaky_example]
 
+        next unless @cache.dependency.key?(example_id)
         next unless (@changed_files & @cache.dependency[example_id]).empty?
 
         @reporter.register_possibly_flaky_example(example_id)
@@ -195,6 +217,7 @@ module RSpecTracer
 
         @filtered_examples[example_id] = EXAMPLE_RUN_REASON[:failed_example]
 
+        next unless @cache.dependency.key?(example_id)
         next unless (@changed_files & @cache.dependency[example_id]).empty?
 
         @reporter.register_possibly_flaky_example(example_id)
@@ -243,6 +266,8 @@ module RSpecTracer
     end
 
     def register_example_files_dependency(example_id)
+      return if @reporter.example_interrupted?(example_id)
+
       example = @reporter.all_examples[example_id]
 
       register_example_file_dependency(example_id, example[:file_name])
@@ -253,6 +278,8 @@ module RSpecTracer
     end
 
     def register_example_file_dependency(example_id, file_name)
+      return if @reporter.example_interrupted?(example_id)
+
       source_file = RSpecTracer::SourceFile.from_name(file_name)
 
       @reporter.register_source_file(source_file)
@@ -260,6 +287,8 @@ module RSpecTracer
     end
 
     def register_file_dependency(example_id, file_path)
+      return if @reporter.example_interrupted?(example_id)
+
       source_file = RSpecTracer::SourceFile.from_path(file_path)
 
       return false if RSpecTracer.filters.any? { |filter| filter.match?(source_file) }

data/lib/rspec_tracer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RSpecTracer
-  VERSION = '0.7.0'
+  VERSION = '0.9.2'
 end

data/lib/rspec_tracer.rb

@@ -28,7 +28,7 @@ require_relative 'rspec_tracer/version'
 
 module RSpecTracer
   class << self
-    attr_accessor :running, :pid
+    attr_accessor :running, :pid, :no_examples
 
     def start(&block)
       RSpecTracer.running = false
@@ -74,7 +74,11 @@ module RSpecTracer
     def at_exit_behavior
       return unless RSpecTracer.pid == Process.pid && RSpecTracer.running
 
+      ::Kernel.exit(1) if runner.incorrect_analysis?
+
       run_exit_tasks
+    ensure
+      RSpecTracer.running = false
     end
 
     def start_example_trace
@@ -154,15 +158,11 @@ module RSpecTracer
     def setup_coverage
       @simplecov = defined?(SimpleCov) && SimpleCov.running
 
-      if simplecov?
-        # rubocop:disable Lint/EmptyBlock
-        SimpleCov.at_exit {}
-        # rubocop:enable Lint/EmptyBlock
-      else
-        require 'coverage'
+      return if simplecov?
 
-        ::Coverage.start
-      end
+      require 'coverage'
+
+      ::Coverage.start
     end
 
     def setup_trace_point
@@ -175,11 +175,13 @@ module RSpecTracer
     end
 
     def run_exit_tasks
-      generate_reports
+      if RSpecTracer.no_examples
+        puts 'Skipped reports generation since all examples were filtered out'
+      else
+        generate_reports
+      end
 
       simplecov? ? run_simplecov_exit_task : run_coverage_exit_task
-    ensure
-      RSpecTracer.running = false
     end
 
     def generate_reports
@@ -194,6 +196,7 @@ module RSpecTracer
     def process_dependency
       starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
+      runner.register_interrupted_examples
       runner.register_deleted_examples
       runner.register_dependency(coverage_reporter.examples_coverage)
       runner.register_untraced_dependency(@traced_files)
@@ -224,12 +227,13 @@ module RSpecTracer
 
       puts 'SimpleCov will now generate coverage report (<3 RSpec tracer)'
 
-      SimpleCov.result.format!
+      coverage_reporter.record_coverage if RSpecTracer.no_examples
     end
 
     def run_coverage_exit_task
       starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
+      coverage_reporter.record_coverage if RSpecTracer.no_examples
       coverage_reporter.generate_final_coverage
 
       file_name = File.join(RSpecTracer.coverage_path, 'coverage.json')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec-tracer
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.2
 platform: ruby
 authors:
 - Abhimanyu Singh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-10 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docile
@@ -50,9 +50,11 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.6.0
-description: RSpec Tracer is a specs dependency analysis tool and a test skipper for
-  RSpec. It maintains a list of files for each test, enabling itself to skip tests
-  in the subsequent runs if none of the dependent files are changed.
+description: RSpec Tracer is a specs dependency analyzer, flaky tests detector, tests
+  accelerator, and coverage reporter tool for RSpec. It maintains a list of files
+  for each test, enabling itself to skip tests in the subsequent runs if none of the
+  dependent files are changed. It uses Ruby's built-in coverage library to keep track
+  of the coverage for each test.
 email:
 - abhisinghabhimanyu@gmail.com
 executables: []
@@ -94,8 +96,10 @@ files:
 - lib/rspec_tracer/html_reporter/views/flaky_examples.erb
 - lib/rspec_tracer/html_reporter/views/layout.erb
 - lib/rspec_tracer/remote_cache/Rakefile
+- lib/rspec_tracer/remote_cache/aws.rb
 - lib/rspec_tracer/remote_cache/cache.rb
-- lib/rspec_tracer/remote_cache/git.rb
+- lib/rspec_tracer/remote_cache/repo.rb
+- lib/rspec_tracer/remote_cache/validator.rb
 - lib/rspec_tracer/reporter.rb
 - lib/rspec_tracer/rspec_reporter.rb
 - lib/rspec_tracer/rspec_runner.rb
@@ -109,7 +113,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/avmnu-sng/rspec-tracer
-  source_code_uri: https://github.com/avmnu-sng/rspec-tracer/tree/v0.7.0
+  source_code_uri: https://github.com/avmnu-sng/rspec-tracer/tree/v0.9.2
   changelog_uri: https://github.com/avmnu-sng/rspec-tracer/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/avmnu-sng/rspec-tracer/issues
 post_install_message:
@@ -130,5 +134,6 @@ requirements: []
 rubygems_version: 3.2.26
 signing_key:
 specification_version: 4
-summary: RSpec Tracer is a specs dependency analysis tool and a test skipper for RSpec
+summary: RSpec Tracer is a specs dependency analyzer, flaky tests detector, tests
+  accelerator, and coverage reporter tool.
 test_files: []