rspec-tracer 0.3.0 → 0.6.1

data/lib/rspec_tracer/html_reporter/views/flaky_examples.erb

@@ -0,0 +1,38 @@
+<div class="report_container" id="<%= title_id %>">
+  <h2>
+    <span class="group_name"><%= title %></span>
+    (
+      <span class="blue">
+        <strong><%= flaky_examples.count %></strong>
+      </span> examples
+    )
+  </h2>
+
+  <a name="<%= title_id %>"></a>
+
+  <div class="report-table--responsive">
+    <table class="report-table">
+      <thead>
+        <tr>
+          <th>ID</th>
+          <th>Description</th>
+          <th>Location</th>
+          <th>Result</th>
+          <th>Run At</th>
+        </tr>
+      </thead>
+
+      <tbody>
+        <% flaky_examples.each do |example| %>
+          <tr>
+            <td><%= example[:id] %></td>
+            <td><%= example[:description] %></td>
+            <td><%= example[:location] %></td>
+            <td><strong class="<%= example_result_css_class(example[:result]) %>"><%= example[:result] %></strong></td>
+            <td width="8%"><%= example[:last_run] %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+</div>

data/lib/rspec_tracer/html_reporter/views/layout.erb

@@ -19,6 +19,9 @@
 
       <div id="content">
         <%= formatted_examples('Examples', examples.values) %>
+        <% unless flaky_examples.empty? %>
+          <%= formatted_flaky_examples('Flaky Examples', flaky_examples) %>
+        <% end %>
         <%= formatted_examples_dependency('Examples Dependency', examples_dependency) %>
         <%= formatted_files_dependency("Files Dependency", files_dependency) %>
       </div>

data/lib/rspec_tracer/remote_cache/cache.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require_relative 'git'
+
+module RSpecTracer
+  module RemoteCache
+    class Cache
+      class CacheDownloadError < StandardError; end
+
+      class CacheUploadError < StandardError; end
+
+      class LocalCacheNotFoundError < StandardError; end
+
+      CACHE_FILES_PER_TEST_SUITE = 8
+
+      def initialize
+        @s3_uri = ENV['RSPEC_TRACER_S3_URI']
+        @aws_s3 = if ENV.fetch('LOCAL_AWS', 'false') == 'true'
+                    'awslocal'
+                  else
+                    'aws'
+                  end
+      end
+
+      def download
+        if @s3_uri.nil?
+          puts 'S3 URI is not configured'
+
+          return
+        end
+
+        prepare_for_download
+
+        if @cache_sha.nil?
+          puts 'Could not find a suitable cache sha to download'
+
+          return
+        end
+
+        download_files
+
+        puts "Downloaded cache from #{@download_prefix} to #{@download_path}"
+      rescue StandardError => e
+        puts "Errored: #{e.message}"
+      end
+
+      def upload
+        if @s3_uri.nil?
+          puts 'S3 URI is not configured'
+
+          return
+        end
+
+        prepare_for_upload
+        upload_files
+
+        puts "Uploaded cache from #{@upload_path} to #{@upload_prefix}"
+      rescue CacheUploadError => e
+        puts "Errored: #{e.message}"
+      end
+
+      private
+
+      def prepare_for_download
+        @test_suite_id = ENV['TEST_SUITE_ID']
+        @test_suites = ENV['TEST_SUITES']
+
+        if @test_suite_id.nil? ^ @test_suites.nil?
+          raise(
+            CacheDownloadError,
+            'Both the enviornment variables TEST_SUITE_ID and TEST_SUITES are not set'
+          )
+        end
+
+        @git = RSpecTracer::RemoteCache::Git.new
+        @git.prepare_for_download
+
+        generate_cached_files_count_and_regex
+
+        @cache_sha = nearest_cache_sha
+      end
+
+      def generate_cached_files_count_and_regex
+        if @test_suites.nil?
+          @last_run_files_count = 1
+          @last_run_files_regex = '/%<ref>s/last_run.json$'
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE
+          @cached_files_regex = '/%<ref>s/[0-9a-f]{32}/.+.json'
+        else
+          @test_suites = @test_suites.to_i
+          @test_suites_regex = (1..@test_suites).to_a.join('|')
+
+          @last_run_files_count = @test_suites
+          @last_run_files_regex = "/%<ref>s/(#{@test_suites_regex})/last_run.json$"
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE * @test_suites.to_i
+          @cached_files_regex = "/%<ref>s/(#{@test_suites_regex})/[0-9a-f]{32}/.+.json$"
+        end
+      end
+
+      def nearest_cache_sha
+        @git.ref_list.detect do |ref|
+          prefix = "#{@s3_uri}/#{ref}/"
+
+          puts "Testing prefix #{prefix}"
+
+          objects = `#{@aws_s3} s3 ls #{prefix} --recursive`.chomp.split("\n")
+
+          last_run_regex = Regexp.new(format(@last_run_files_regex, ref: ref))
+
+          next if objects.count { |object| object.match?(last_run_regex) } != @last_run_files_count
+
+          cache_regex = Regexp.new(format(@cached_files_regex, ref: ref))
+
+          objects.count { |object| object.match?(cache_regex) } == @cached_files_count
+        end
+      end
+
+      def download_files
+        @download_prefix = "#{@s3_uri}/#{@cache_sha}/#{@test_suite_id}/".sub(%r{/+$}, '/')
+        @download_path = RSpecTracer.cache_path
+
+        raise CacheDownloadError, 'Failed to download cache files' unless system(
+          @aws_s3, 's3', 'cp',
+          File.join(@download_prefix, 'last_run.json'),
+          @download_path,
+          out: File::NULL, err: File::NULL
+        )
+
+        @run_id = last_run_id
+
+        return if system(
+          @aws_s3, 's3', 'cp',
+          File.join(@download_prefix, @run_id),
+          File.join(@download_path, @run_id),
+          '--recursive',
+          out: File::NULL, err: File::NULL
+        )
+
+        FileUtils.rm_rf(@download_path)
+
+        raise CacheDownloadError, 'Failed to download cache files'
+      end
+
+      def prepare_for_upload
+        @git = RSpecTracer::RemoteCache::Git.new
+        @test_suite_id = ENV['TEST_SUITE_ID']
+        @upload_prefix = if @test_suite_id.nil?
+                           "#{@s3_uri}/#{@git.branch_ref}/"
+                         else
+                           "#{@s3_uri}/#{@git.branch_ref}/#{@test_suite_id}/"
+                         end
+
+        @upload_path = RSpecTracer.cache_path
+        @run_id = last_run_id
+      end
+
+      def upload_files
+        return if system(
+          @aws_s3, 's3', 'cp',
+          File.join(@upload_path, 'last_run.json'),
+          @upload_prefix,
+          out: File::NULL, err: File::NULL
+        ) && system(
+          @aws_s3, 's3', 'cp',
+          File.join(@upload_path, @run_id),
+          File.join(@upload_prefix, @run_id),
+          '--recursive',
+          out: File::NULL, err: File::NULL
+        )
+
+        raise CacheUploadError, 'Failed to upload cache files'
+      end
+
+      def last_run_id
+        file_name = File.join(RSpecTracer.cache_path, 'last_run.json')
+
+        return unless File.file?(file_name)
+
+        run_id = JSON.parse(File.read(file_name))['run_id']
+
+        raise LocalCacheNotFoundError, 'Could not find any local cache to upload' if run_id.nil?
+
+        run_id
+      end
+    end
+  end
+end

data/lib/rspec_tracer/remote_cache/git.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  module RemoteCache
+    class Git
+      class GitOperationError < StandardError; end
+
+      attr_reader :branch_ref, :ref_list
+
+      def initialize
+        fetch_head_ref
+        fetch_branch_ref
+      end
+
+      def prepare_for_download
+        fetch_unreachable_refs
+        fetch_ancestry_refs
+        fetch_ordered_refs
+      end
+
+      private
+
+      def fetch_head_ref
+        @head_ref = `git rev-parse HEAD`.chomp
+
+        raise GitOperationError, 'Could not find HEAD commit sha' unless $CHILD_STATUS.success?
+      end
+
+      def fetch_branch_ref
+        @merged_parents = []
+        @ignored_refs = []
+
+        unless merged?
+          @branch_ref = @head_ref
+
+          return
+        end
+
+        @ignored_refs << @head_ref
+
+        fetch_merged_parents
+        fetch_merged_branch_ref
+      end
+
+      def merged?
+        system('git', 'rev-parse', 'HEAD^2', out: File::NULL, err: File::NULL)
+      end
+
+      def fetch_merged_parents
+        first_parent = `git rev-parse HEAD^1`.chomp
+        @merged_parents << first_parent if $CHILD_STATUS.success?
+
+        second_parent = `git rev-parse HEAD^2`.chomp
+        @merged_parents << second_parent if $CHILD_STATUS.success?
+
+        raise GitOperationError, 'Could not find merged commit parents' if @merged_parents.length != 2
+      end
+
+      def fetch_merged_branch_ref
+        @origin_head_ref = `git rev-parse origin/HEAD`.chomp
+        @branch_ref = nil
+
+        if @merged_parents.first != @origin_head_ref
+          @branch_ref = @head_ref
+          @ignored_refs = []
+
+          return
+        end
+
+        @branch_ref = @merged_parents.last
+        @ignored_refs = @ignored_refs.to_set | `git rev-list #{@branch_ref}..origin/HEAD`.chomp.split
+
+        raise GitOperationError, 'Could not find ignored refs' unless $CHILD_STATUS.success?
+      end
+
+      def fetch_unreachable_refs
+        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
+          git fsck
+              --no-progress
+              --unreachable
+              --connectivity-only #{@branch_ref}
+            | awk '/commit/ { print $3 }'
+            | head -n 25
+        COMMAND
+
+        @unreachable_refs = `#{command}`.chomp.split
+
+        raise GitOperationError, 'Could not find unreachable refs' unless $CHILD_STATUS.success?
+      end
+
+      def fetch_ancestry_refs
+        @ancestry_refs = `git rev-list --max-count=25 #{@branch_ref}`.chomp.split
+
+        raise GitOperationError, 'Could not find ancestry refs' unless $CHILD_STATUS.success?
+      end
+
+      def fetch_ordered_refs
+        unordered_refs = (@unreachable_refs.to_set | @ancestry_refs) - @ignored_refs
+
+        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
+          git rev-list
+            --topo-order
+            --no-walk=sorted
+            #{unordered_refs.to_a.join(' ')}
+        COMMAND
+
+        @ref_list = `#{command}`.chomp.split
+
+        raise GitOperationError, 'Could not find refs to download cache' unless $CHILD_STATUS.success?
+      end
+    end
+  end
+end

data/lib/rspec_tracer/reporter.rb

@@ -2,8 +2,9 @@
 
 module RSpecTracer
   class Reporter
-    attr_reader :all_examples, :pending_examples, :all_files, :dependency,
-                :reverse_dependency, :examples_coverage, :last_run
+    attr_reader :all_examples, :possibly_flaky_examples, :flaky_examples, :pending_examples,
+                :all_files, :modified_files, :deleted_files, :dependency, :reverse_dependency,
+                :examples_coverage, :last_run
 
     def initialize
       initialize_examples
@@ -21,6 +22,7 @@ module RSpecTracer
     end
 
     def on_example_passed(example_id, result)
+      @passed_examples << example_id
       @all_examples[example_id][:execution_result] = formatted_execution_result(result)
     end
 
@@ -44,6 +46,14 @@ module RSpecTracer
       end
     end
 
+    def register_possibly_flaky_example(example_id)
+      @possibly_flaky_examples << example_id
+    end
+
+    def register_flaky_example(example_id)
+      @flaky_examples << example_id
+    end
+
     def register_failed_example(example_id)
       @failed_examples << example_id
     end
@@ -52,6 +62,10 @@ module RSpecTracer
       @pending_examples << example_id
     end
 
+    def example_passed?(example_id)
+      @passed_examples.include?(example_id)
+    end
+
     def example_skipped?(example_id)
       @skipped_examples.include?(example_id)
     end
@@ -126,6 +140,8 @@ module RSpecTracer
     end
 
     def write_reports
+      starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
       @run_id = Digest::MD5.hexdigest(@all_examples.keys.sort.to_json)
       @cache_dir = File.join(RSpecTracer.cache_path, @run_id)
 
@@ -133,6 +149,7 @@ module RSpecTracer
 
       %i[
         all_examples
+        flaky_examples
         failed_examples
         pending_examples
         all_files
@@ -142,13 +159,19 @@ module RSpecTracer
         last_run
       ].each { |report_type| send("write_#{report_type}_report") }
 
-      puts "RSpec tracer reports generated to #{@cache_dir}"
+      ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      elpased = RSpecTracer::TimeFormatter.format_time(ending - starting)
+
+      puts "RSpec tracer reports written to #{@cache_dir} (took #{elpased})"
     end
 
     private
 
     def initialize_examples
       @all_examples = {}
+      @passed_examples = Set.new
+      @possibly_flaky_examples = Set.new
+      @flaky_examples = Set.new
       @failed_examples = Set.new
       @skipped_examples = Set.new
       @pending_examples = Set.new
@@ -206,50 +229,56 @@ module RSpecTracer
     def write_all_examples_report
       file_name = File.join(@cache_dir, 'all_examples.json')
 
-      File.write(file_name, JSON.pretty_generate(@all_examples))
+      File.write(file_name, JSON.generate(@all_examples))
+    end
+
+    def write_flaky_examples_report
+      file_name = File.join(@cache_dir, 'flaky_examples.json')
+
+      File.write(file_name, JSON.generate(@flaky_examples.to_a))
     end
 
     def write_failed_examples_report
       file_name = File.join(@cache_dir, 'failed_examples.json')
 
-      File.write(file_name, JSON.pretty_generate(@failed_examples.to_a))
+      File.write(file_name, JSON.generate(@failed_examples.to_a))
     end
 
     def write_pending_examples_report
       file_name = File.join(@cache_dir, 'pending_examples.json')
 
-      File.write(file_name, JSON.pretty_generate(@pending_examples.to_a))
+      File.write(file_name, JSON.generate(@pending_examples.to_a))
     end
 
     def write_all_files_report
       file_name = File.join(@cache_dir, 'all_files.json')
 
-      File.write(file_name, JSON.pretty_generate(@all_files))
+      File.write(file_name, JSON.generate(@all_files))
     end
 
     def write_dependency_report
       file_name = File.join(@cache_dir, 'dependency.json')
 
-      File.write(file_name, JSON.pretty_generate(@dependency))
+      File.write(file_name, JSON.generate(@dependency))
     end
 
     def write_reverse_dependency_report
       file_name = File.join(@cache_dir, 'reverse_dependency.json')
 
-      File.write(file_name, JSON.pretty_generate(@reverse_dependency))
+      File.write(file_name, JSON.generate(@reverse_dependency))
     end
 
     def write_examples_coverage_report
       file_name = File.join(@cache_dir, 'examples_coverage.json')
 
-      File.write(file_name, JSON.pretty_generate(@examples_coverage))
+      File.write(file_name, JSON.generate(@examples_coverage))
     end
 
     def write_last_run_report
       file_name = File.join(RSpecTracer.cache_path, 'last_run.json')
       last_run_data = @last_run.merge(run_id: @run_id, timestamp: Time.now.utc)
 
-      File.write(file_name, JSON.pretty_generate(last_run_data))
+      File.write(file_name, JSON.generate(last_run_data))
     end
   end
 end