rspec-tracer 0.7.0 → 0.8.0

data/lib/rspec_tracer/remote_cache/repo.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  module RemoteCache
+    class Repo
+      class RepoError < StandardError; end
+
+      attr_reader :branch_name, :branch_ref, :branch_refs, :ancestry_refs, :cache_refs
+
+      def initialize(aws)
+        @aws = aws
+        @branch_name = ENV['GIT_BRANCH'].chomp
+
+        raise RepoError, 'GIT_BRANCH environment variable is not set' if @branch_name.nil?
+
+        fetch_head_ref
+        fetch_branch_ref
+        fetch_ancestry_refs
+        fetch_branch_refs
+        generate_cache_refs
+      end
+
+      private
+
+      def fetch_head_ref
+        @head_ref = `git rev-parse HEAD`.chomp
+
+        raise RepoError, 'Could not find HEAD commit sha' unless $CHILD_STATUS.success?
+      end
+
+      def fetch_branch_ref
+        @merged_parents = []
+        @ignored_refs = []
+
+        unless merged?
+          @branch_ref = @head_ref
+
+          return
+        end
+
+        @ignored_refs << @head_ref
+
+        fetch_merged_parents
+        fetch_merged_branch_ref
+      end
+
+      def fetch_ancestry_refs
+        ref_list = `git rev-list --max-count=25 #{@branch_ref}`.chomp.split
+
+        raise RepoError, 'Could not find ancestry refs' unless $CHILD_STATUS.success?
+
+        ref_list = ref_list.to_set - @ignored_refs
+        @ancestry_refs = refs_committer_timestamp(ref_list.to_a)
+
+        return if @ancestry_refs.empty?
+
+        print_refs(@ancestry_refs, 'ancestry')
+      end
+
+      def fetch_branch_refs
+        unless @aws.branch_refs?(@branch_name)
+          puts "No branch refs for #{@branch_name} branch found in S3"
+
+          @branch_refs = {}
+
+          return
+        end
+
+        download_branch_refs
+      end
+
+      def generate_cache_refs
+        ref_list = @ancestry_refs.merge(@branch_refs)
+
+        if ref_list.empty?
+          @cache_refs = {}
+
+          return
+        end
+
+        @cache_refs = ref_list.sort_by { |_, timestamp| -timestamp }.to_h
+
+        print_refs(@cache_refs, 'cache')
+      end
+
+      def merged?
+        system('git', 'rev-parse', 'HEAD^2', out: File::NULL, err: File::NULL)
+      end
+
+      def fetch_merged_parents
+        first_parent = `git rev-parse HEAD^1`.chomp
+        @merged_parents << first_parent if $CHILD_STATUS.success?
+
+        second_parent = `git rev-parse HEAD^2`.chomp
+        @merged_parents << second_parent if $CHILD_STATUS.success?
+
+        raise RepoError, 'Could not find merged commit parents' if @merged_parents.length != 2
+      end
+
+      def fetch_merged_branch_ref
+        @origin_head_ref = `git rev-parse origin/HEAD`.chomp
+        @branch_ref = nil
+
+        if @merged_parents.first != @origin_head_ref
+          @branch_ref = @head_ref
+          @ignored_refs = []
+
+          return
+        end
+
+        @branch_ref = @merged_parents.last
+        @ignored_refs = @ignored_refs.to_set | `git rev-list #{@branch_ref}..origin/HEAD`.chomp.split
+
+        raise RepoError, 'Could not find ignored refs' unless $CHILD_STATUS.success?
+      end
+
+      def refs_committer_timestamp(ref_list)
+        return {} if ref_list.empty?
+
+        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
+          git show
+            --no-patch
+            --format="%H %ct"
+            #{ref_list.join(' ')}
+        COMMAND
+
+        ref_list = `#{command}`.chomp
+
+        raise RepoError, 'Could not find ancestry refs' unless $CHILD_STATUS.success?
+
+        ref_list.split("\n").map(&:split).to_h.transform_values(&:to_i)
+      end
+
+      def download_branch_refs
+        file_name = File.join(RSpecTracer.cache_path, 'branch_refs.json')
+
+        if @aws.download_branch_refs(branch_name, file_name)
+          @branch_refs = JSON.parse(File.read(file_name)).transform_values(&:to_i)
+
+          return if @branch_refs.empty?
+
+          filter_branch_refs
+          print_refs(@branch_refs, 'branch')
+        else
+          @branch_refs = {}
+
+          File.rm_f(file_name)
+
+          puts "Failed to fetch branch refs for #{@branch_name} branch"
+        end
+      end
+
+      def filter_branch_refs
+        if @ancestry_refs.empty?
+          @branch_refs = @branch_refs.sort_by { |_, timestamp| -timestamp }.first(25).to_h
+
+          return
+        end
+
+        oldest_ancestry_time = @ancestry_refs.values.min
+
+        @branch_refs = @branch_refs
+          .select { |_, timestamp| timestamp >= oldest_ancestry_time }
+          .sort_by { |_, timestamp| -timestamp }
+          .first(25)
+          .to_h
+      end
+
+      def print_refs(refs, type)
+        puts "Fetched the following #{type} refs for #{@branch_name} branch:"
+        puts refs.map { |ref, timestamp| "  * #{ref} (commit timestamp: #{timestamp})" }.join("\n")
+      end
+    end
+  end
+end

data/lib/rspec_tracer/remote_cache/validator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module RSpecTracer
+  module RemoteCache
+    class Validator
+      CACHE_FILES_PER_TEST_SUITE = 8
+
+      def initialize
+        @test_suite_id = ENV['TEST_SUITE_ID']
+        @test_suites = ENV['TEST_SUITES']
+
+        if @test_suite_id.nil? ^ @test_suites.nil?
+          raise(
+            ValidationError,
+            'Both the enviornment variables TEST_SUITE_ID and TEST_SUITES are not set'
+          )
+        end
+
+        setup
+      end
+
+      def valid?(ref, cache_files)
+        last_run_regex = Regexp.new(format(@last_run_files_regex, ref: ref))
+
+        return false if cache_files.count { |file| file.match?(last_run_regex) } != @last_run_files_count
+
+        cache_regex = Regexp.new(format(@cached_files_regex, ref: ref))
+
+        cache_files.count { |file| file.match?(cache_regex) } == @cached_files_count
+      end
+
+      private
+
+      def setup
+        if @test_suites.nil?
+          @last_run_files_count = 1
+          @last_run_files_regex = '/%<ref>s/last_run.json$'
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE
+          @cached_files_regex = '/%<ref>s/[0-9a-f]{32}/.+.json'
+        else
+          @test_suites = @test_suites.to_i
+          @test_suites_regex = (1..@test_suites).to_a.join('|')
+
+          @last_run_files_count = @test_suites
+          @last_run_files_regex = "/%<ref>s/(#{@test_suites_regex})/last_run.json$"
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE * @test_suites
+          @cached_files_regex = "/%<ref>s/(#{@test_suites_regex})/[0-9a-f]{32}/.+.json$"
+        end
+      end
+    end
+  end
+end

data/lib/rspec_tracer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RSpecTracer
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec-tracer
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Abhimanyu Singh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-10 00:00:00.000000000 Z
+date: 2021-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docile
@@ -94,8 +94,10 @@ files:
 - lib/rspec_tracer/html_reporter/views/flaky_examples.erb
 - lib/rspec_tracer/html_reporter/views/layout.erb
 - lib/rspec_tracer/remote_cache/Rakefile
+- lib/rspec_tracer/remote_cache/aws.rb
 - lib/rspec_tracer/remote_cache/cache.rb
-- lib/rspec_tracer/remote_cache/git.rb
+- lib/rspec_tracer/remote_cache/repo.rb
+- lib/rspec_tracer/remote_cache/validator.rb
 - lib/rspec_tracer/reporter.rb
 - lib/rspec_tracer/rspec_reporter.rb
 - lib/rspec_tracer/rspec_runner.rb
@@ -109,7 +111,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/avmnu-sng/rspec-tracer
-  source_code_uri: https://github.com/avmnu-sng/rspec-tracer/tree/v0.7.0
+  source_code_uri: https://github.com/avmnu-sng/rspec-tracer/tree/v0.8.0
   changelog_uri: https://github.com/avmnu-sng/rspec-tracer/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/avmnu-sng/rspec-tracer/issues
 post_install_message:

data/lib/rspec_tracer/remote_cache/git.rb

@@ -1,113 +0,0 @@
-# frozen_string_literal: true
-
-module RSpecTracer
-  module RemoteCache
-    class Git
-      class GitOperationError < StandardError; end
-
-      attr_reader :branch_ref, :ref_list
-
-      def initialize
-        fetch_head_ref
-        fetch_branch_ref
-      end
-
-      def prepare_for_download
-        fetch_unreachable_refs
-        fetch_ancestry_refs
-        fetch_ordered_refs
-      end
-
-      private
-
-      def fetch_head_ref
-        @head_ref = `git rev-parse HEAD`.chomp
-
-        raise GitOperationError, 'Could not find HEAD commit sha' unless $CHILD_STATUS.success?
-      end
-
-      def fetch_branch_ref
-        @merged_parents = []
-        @ignored_refs = []
-
-        unless merged?
-          @branch_ref = @head_ref
-
-          return
-        end
-
-        @ignored_refs << @head_ref
-
-        fetch_merged_parents
-        fetch_merged_branch_ref
-      end
-
-      def merged?
-        system('git', 'rev-parse', 'HEAD^2', out: File::NULL, err: File::NULL)
-      end
-
-      def fetch_merged_parents
-        first_parent = `git rev-parse HEAD^1`.chomp
-        @merged_parents << first_parent if $CHILD_STATUS.success?
-
-        second_parent = `git rev-parse HEAD^2`.chomp
-        @merged_parents << second_parent if $CHILD_STATUS.success?
-
-        raise GitOperationError, 'Could not find merged commit parents' if @merged_parents.length != 2
-      end
-
-      def fetch_merged_branch_ref
-        @origin_head_ref = `git rev-parse origin/HEAD`.chomp
-        @branch_ref = nil
-
-        if @merged_parents.first != @origin_head_ref
-          @branch_ref = @head_ref
-          @ignored_refs = []
-
-          return
-        end
-
-        @branch_ref = @merged_parents.last
-        @ignored_refs = @ignored_refs.to_set | `git rev-list #{@branch_ref}..origin/HEAD`.chomp.split
-
-        raise GitOperationError, 'Could not find ignored refs' unless $CHILD_STATUS.success?
-      end
-
-      def fetch_unreachable_refs
-        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
-          git fsck
-              --no-progress
-              --unreachable
-              --connectivity-only #{@branch_ref}
-            | awk '/commit/ { print $3 }'
-            | head -n 25
-        COMMAND
-
-        @unreachable_refs = `#{command}`.chomp.split
-
-        raise GitOperationError, 'Could not find unreachable refs' unless $CHILD_STATUS.success?
-      end
-
-      def fetch_ancestry_refs
-        @ancestry_refs = `git rev-list --max-count=25 #{@branch_ref}`.chomp.split
-
-        raise GitOperationError, 'Could not find ancestry refs' unless $CHILD_STATUS.success?
-      end
-
-      def fetch_ordered_refs
-        unordered_refs = (@unreachable_refs.to_set | @ancestry_refs) - @ignored_refs
-
-        command = <<-COMMAND.strip.gsub(/\s+/, ' ')
-          git rev-list
-            --topo-order
-            --no-walk=sorted
-            #{unordered_refs.to_a.join(' ')}
-        COMMAND
-
-        @ref_list = `#{command}`.chomp.split
-
-        raise GitOperationError, 'Could not find refs to download cache' unless $CHILD_STATUS.success?
-      end
-    end
-  end
-end