rspec-tracer 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34f47b6527f2c8f3c1cb3524adf8d81a9fe7e3833c202605a206598dee8758b2
-  data.tar.gz: e647137006f5f8af18999ea8f2665c09d703bf2cd8ea1e84dec85e66f87ba5e2
+  metadata.gz: 2817cccf4825e28177bb8794f8d261635f81ec5c1bd6ca0242ba171e702850ca
+  data.tar.gz: 9c5f6af579eec43a994c28b5e8132fec5d71f7a2ddddfb531fafd5c51943d283
 SHA512:
-  metadata.gz: 05c204bd00cfa15258a1770790360184c9481d670cc8ef3542b06024976a9e13cf3595e79ecefa0d22c8337d35dee9ef6963b145cf1aa16268f3de467c7d171b
-  data.tar.gz: b2f7e88fbf404793a9e2daaade7448d51bd63150701a6735c9ae8bdcb17eaf25c926bde411cd61bc5dd20133829d1576d16b7588336f05ec968ae4f3d82a5453
+  metadata.gz: 2d3af598e09b8ac043b5f605acf39b588701c4f0e7b4d74646eb0da74c1c85746db60f215e02cd5fba1880bb75288d0d35df3986b08cc71cfbad4d45ad722d2f
+  data.tar.gz: 7ce36336213d4d37337afa5b50fd774bd6ef7b235539f3a289f1a804a878fba3fa1e7972e9bd09bb8083c80929c0fc3b6bf0b30f257542fdb5e5c466c1247360

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.5.0] - 2021-09-03
+
+### Fixed
+
+- Limit number of cached files download (#16)
+
 ## [0.4.0] - 2021-09-03
 
 ### Added

data/README.md

@@ -33,8 +33,8 @@ and **caching on CI**, etc.
   * [RSPEC_TRACER_NO_SKIP](#rspec_tracer_no_skip)
   * [RSPEC_TRACER_S3_URI](#rspec_tracer_s3_uri)
   * [RSPEC_TRACER_UPLOAD_LOCAL_CACHE](#rspec_tracer_upload_local_cache)
-  * [TEST_SUITE_ID](#test_suite_id)
   * [TEST_SUITES](#test_suites)
+  * [TEST_SUITE_ID](#test_suite_id)
 * [Sample Reports](#sample-reports)
     * [Examples](#examples)
     * [Examples Dependency](#examples-dependency)
@@ -128,13 +128,13 @@ Rakefile in your project to have the following:
     ```
 3. Before running tests, download the remote cache using the following rake task:
 
-    ```ruby
+    ```sh
     bundle exec rake rspec_tracer:remote_cache:download
     ```
 4. Run the tests with RSpec using `bundle exec rspec`.
 5. After running tests, upload the local cache using the following rake task:
 
-    ```ruby
+    ```sh
     bundle exec rake rspec_tracer:remote_cache:upload
     ```
 6. After running your tests, open `rspec_tracer_report/index.html` in the
@@ -177,6 +177,15 @@ export RSPEC_TRACER_S3_URI=s3://ci-artifacts-bucket/rspec-tracer-cache
 By default, RSpec Tracer does not upload local cache files. You can set this
 environment variable to `true` to upload the local cache to S3.
 
+### TEST_SUITES
+
+Set this environment variable when using test suite id. It determines the total
+number of different test suites you are running.
+
+```ruby
+export TEST_SUITES=8
+```
+
 ### TEST_SUITE_ID
 
 If you have a large set of tests to run, it is recommended to run them in
@@ -189,13 +198,19 @@ TEST_SUITE_ID=1 bundle exec rspec spec/models
 TEST_SUITE_ID=2 bundle exec rspec spec/helpers
 ```
 
-### TEST_SUITES
+If you run parallel builds on the CI, you should specify the test suite ID and
+the total number of test suites when downloading the cache files.
 
-Set this environment variable when using test suite id. It determines the total
-number of different test suites you are running.
+```sh
+$ TEST_SUITES=5 TEST_SUITE_ID=1 bundle exec rake rspec_tracer:remote_cache:download
+```
 
-```ruby
-export TEST_SUITES=8
+In this case, the appropriate cache should have all the cache files available on
+the S3 for each test suite, not just for the current one. Also, while uploading,
+make sure to provide the test suite id.
+
+```sh
+$ TEST_SUITE_ID=1 bundle exec rake rspec_tracer:remote_cache:upload
 ```
 
 ## Sample Reports

data/lib/rspec_tracer/remote_cache/cache.rb

@@ -11,7 +11,7 @@ module RSpecTracer
 
       class LocalCacheNotFoundError < StandardError; end
 
-      CACHE_FILES_PER_TEST_SUITE = 8
+      CACHE_FILES_PER_TEST_SUITE = 7
 
       def initialize
         @s3_uri = ENV['RSPEC_TRACER_S3_URI']
@@ -20,9 +20,6 @@ module RSpecTracer
                   else
                     'aws'
                   end
-        @test_suite_id = ENV['TEST_SUITE_ID'].to_s
-        @test_suites = ENV.fetch('TEST_SUITES', '1').to_i
-        @total_objects = CACHE_FILES_PER_TEST_SUITE * @test_suites
       end
 
       def download
@@ -32,10 +29,7 @@ module RSpecTracer
           return
         end
 
-        @git = RSpecTracer::RemoteCache::Git.new
-        @git.prepare_for_download
-
-        @cache_sha = nearest_cache_sha
+        prepare_for_download
 
         if @cache_sha.nil?
           puts 'Could not find a suitable cache sha to download'
@@ -57,9 +51,7 @@ module RSpecTracer
           return
         end
 
-        @run_id = last_run_id
-        @git = RSpecTracer::RemoteCache::Git.new
-
+        prepare_for_upload
         upload_files
 
         puts "Uploaded cache from #{@upload_path} to #{@upload_prefix}"
@@ -69,20 +61,57 @@ module RSpecTracer
 
       private
 
+      def prepare_for_download
+        @test_suite_id = ENV['TEST_SUITE_ID']
+        @test_suites = ENV['TEST_SUITES']
+
+        if @test_suite_id.nil? ^ @test_suites.nil?
+          raise(
+            CacheDownloadError,
+            'Both the enviornment variables TEST_SUITE_ID and TEST_SUITES are not set'
+          )
+        end
+
+        @git = RSpecTracer::RemoteCache::Git.new
+        @git.prepare_for_download
+
+        generate_cached_files_count_and_regex
+
+        @cache_sha = nearest_cache_sha
+      end
+
+      def generate_cached_files_count_and_regex
+        if @test_suites.nil?
+          @last_run_files_count = 1
+          @last_run_files_regex = '/%<ref>s/last_run.json$'
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE
+          @cached_files_regex = '/%<ref>s/[0-9a-f]{32}/.+.json'
+        else
+          @test_suites = @test_suites.to_i
+          @test_suites_regex = (1..@test_suites).to_a.join('|')
+
+          @last_run_files_count = @test_suites
+          @last_run_files_regex = "/%<ref>s/(#{@test_suites_regex})/last_run.json$"
+          @cached_files_count = CACHE_FILES_PER_TEST_SUITE * @test_suites.to_i
+          @cached_files_regex = "/%<ref>s/(#{@test_suites_regex})/[0-9a-f]{32}/.+.json$"
+        end
+      end
+
       def nearest_cache_sha
         @git.ref_list.detect do |ref|
-          prefix = "#{@s3_uri}/#{ref}/#{@test_suite_id}/".sub(%r{/+$}, '/')
+          prefix = "#{@s3_uri}/#{ref}/"
 
           puts "Testing prefix #{prefix}"
 
-          command = <<-COMMAND.strip.gsub(/\s+/, ' ')
-            #{@aws_s3} s3 ls #{prefix}
-              --recursive
-              --summarize
-            | grep 'Total Objects'
-          COMMAND
+          objects = `#{@aws_s3} s3 ls #{prefix} --recursive`.chomp.split("\n")
+
+          last_run_regex = Regexp.new(format(@last_run_files_regex, ref: ref))
+
+          next if objects.count { |object| object.match?(last_run_regex) } != @last_run_files_count
 
-          @total_objects == `#{command}`.chomp.split('Total Objects:').last.to_s.strip.to_i
+          cache_regex = Regexp.new(format(@cached_files_regex, ref: ref))
+
+          objects.count { |object| object.match?(cache_regex) } == @cached_files_count
         end
       end
 
@@ -90,10 +119,19 @@ module RSpecTracer
         @download_prefix = "#{@s3_uri}/#{@cache_sha}/#{@test_suite_id}/".sub(%r{/+$}, '/')
         @download_path = RSpecTracer.cache_path
 
-        return if system(
+        raise CacheDownloadError, 'Failed to download cache files' unless system(
           @aws_s3, 's3', 'cp',
-          @download_prefix,
+          File.join(@download_prefix, 'last_run.json'),
           @download_path,
+          out: File::NULL, err: File::NULL
+        )
+
+        @run_id = last_run_id
+
+        return if system(
+          @aws_s3, 's3', 'cp',
+          File.join(@download_prefix, @run_id),
+          File.join(@download_path, @run_id),
           '--recursive',
           out: File::NULL, err: File::NULL
         )
@@ -103,22 +141,20 @@ module RSpecTracer
         raise CacheDownloadError, 'Failed to download cache files'
       end
 
-      def last_run_id
-        file_name = File.join(RSpecTracer.cache_path, 'last_run.json')
-
-        return unless File.file?(file_name)
-
-        run_id = JSON.parse(File.read(file_name))['run_id']
-
-        raise LocalCacheNotFoundError, 'Could not find any local cache to upload' if run_id.nil?
+      def prepare_for_upload
+        @git = RSpecTracer::RemoteCache::Git.new
+        @test_suite_id = ENV['TEST_SUITE_ID']
+        @upload_prefix = if @test_suite_id.nil?
+                           "#{@s3_uri}/#{@git.branch_ref}/"
+                         else
+                           "#{@s3_uri}/#{@git.branch_ref}/#{@test_suite_id}/"
+                         end
 
-        run_id
+        @upload_path = RSpecTracer.cache_path
+        @run_id = last_run_id
       end
 
       def upload_files
-        @upload_prefix = "#{@s3_uri}/#{@git.branch_ref}/#{@test_suite_id}/".sub(%r{/+$}, '/')
-        @upload_path = RSpecTracer.cache_path
-
         return if system(
           @aws_s3, 's3', 'cp',
           File.join(@upload_path, 'last_run.json'),
@@ -127,13 +163,25 @@ module RSpecTracer
         ) && system(
           @aws_s3, 's3', 'cp',
           File.join(@upload_path, @run_id),
-          "#{@upload_prefix}/#{@run_id}",
+          File.join(@upload_prefix, @run_id),
           '--recursive',
           out: File::NULL, err: File::NULL
         )
 
         raise CacheUploadError, 'Failed to upload cache files'
       end
+
+      def last_run_id
+        file_name = File.join(RSpecTracer.cache_path, 'last_run.json')
+
+        return unless File.file?(file_name)
+
+        run_id = JSON.parse(File.read(file_name))['run_id']
+
+        raise LocalCacheNotFoundError, 'Could not find any local cache to upload' if run_id.nil?
+
+        run_id
+      end
     end
   end
 end

data/lib/rspec_tracer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RSpecTracer
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec-tracer
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Abhimanyu Singh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-02 00:00:00.000000000 Z
+date: 2021-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docile
@@ -105,7 +105,7 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/avmnu-sng/rspec-tracer
-  source_code_uri: https://github.com/avmnu-sng/rspec-tracer/tree/v0.4.0
+  source_code_uri: https://github.com/avmnu-sng/rspec-tracer/tree/v0.5.0
   changelog_uri: https://github.com/avmnu-sng/rspec-tracer/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/avmnu-sng/rspec-tracer/issues
 post_install_message: