quarantine 2.1.2 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5b572688cff1561139754374abf14bfc0fbbce203dd7ab8d0db5916b2c5b67ba
-  data.tar.gz: 8dfc948827ad091c3424dd425ba157294340fff95ec603e76f1ffcc5059541bd
+  metadata.gz: 2d156442ce5b26736708dbc75881cf1ba37776d69970a72f661ea10d77448d8f
+  data.tar.gz: 576657a82b094339bd91ca64ad9467ed7774ce1e45636d4e5051209f7bf52a6d
 SHA512:
-  metadata.gz: 879da05b1135ab244f4246e8003418a0e5b50f68ac7ca408b7ead93ebc44b5b54676d84be4b6793166c84ddf4816f92c09664ad6d78cc02755c7bf4c0de181b9
-  data.tar.gz: d7d9cc51559136896dde5ad0ebde281542cbe2ed4b0a42a7dfa508a264d070bab67d1ea9dfe1c213e50fa98308e85430791d69e57b301c1d1d11ce3883bcfeab
+  metadata.gz: 58eb71c5b9bdc37d0561f49c3e6aaf93407de7f00f76c0823bbf84388853d6c2bf413c72c4f59aba737ba271380f055e30205daf80db337edc0f186bf2f61ca5
+  data.tar.gz: 2ce73d002d65bf972228aeaada7bb1cba24a21ff14da8414f112238e8abac6852b198eff9cbb0d146d863e4a48718d69627bf592c61669067d9bd09fe5c9b5c3

data/README.md

@@ -101,16 +101,25 @@ Quarantine comes with built-in support for the following database types:
 
 To use `:dynamodb`, be sure to add `gem 'aws-sdk-dynamodb', '~> 1', group: :test` to your `Gemfile`.
 
-To use `:google_sheets`, be sure to add `gem 'google_drive', '~> 2', group: :test` to your `Gemfile`. Here's an example:
+To use `:google_sheets`, be sure to add `gem 'google_drive', '~> 3', group: :test` to your `Gemfile`. Here's an example:
 
 ```rb
 config.quarantine_database = {
   type: :google_sheets,
   authorization: {type: :service_account_key, file: "service_account.json"}, # also accepts `type: :config`
-  spreadsheet: {type: :by_key, "1Jb5fC6wSuIMnP85tUR5knuZ4f5fuu4nMzQF6-0l-EXAMPLE"}, # also accepts `type: :by_title` and `type: :by_url`
+  spreadsheet: {
+    type: :by_key, # also accepts `type: :by_title` and `type: :by_url`
+    key: "1Jb5fC6wSuIMnP85tUR5knuZ4f5fuu4nMzQF6-0l-EXAMPLE"}, # also accepts `type: :by_title` and `type: :by_url`
 }
 ```
 
+The spreadsheet first line (1) should contains: id, full_description, updated_at, last_status, location, extra_attributes, consecutive_passes. Something like:
+
+ A | B                   | C          | D           | E        | F                | G
+-- | :-----------------: | :---------:| :----------:| :-------:| :---------------:| :--:
+id | full_description    | updated_at | last_status | location | extra_attributes | consecutive_passes
+
+
 To use a custom database that's not provided, subclass `Quarantine::Databases::Base` and pass an instance of your class as the `quarantine_database` setting:
 
 ```rb

data/lib/quarantine.rb

@@ -9,23 +9,6 @@ require 'quarantine/databases/base'
 require 'quarantine/databases/dynamo_db'
 require 'quarantine/databases/google_sheets'
 
-module RSpec
-  module Core
-    class Example
-      extend T::Sig
-
-      # The implementation of clear_exception in rspec-retry doesn't work
-      # for examples that use `it_behaves_like`, so we implement our own version that
-      # clear the exception field recursively.
-      sig { void }
-      def clear_exception!
-        @exception = T.let(nil, T.untyped)
-        T.unsafe(self).example.clear_exception! if defined?(example)
-      end
-    end
-  end
-end
-
 class Quarantine
   extend T::Sig
 
@@ -66,7 +49,7 @@ class Quarantine
 
   # Scans the test_statuses from the database and store their IDs in quarantined_ids
   sig { void }
-  def fetch_test_statuses
+  def on_start
     begin
       test_statuses = database.fetch_items(@options[:test_statuses_table_name])
     rescue Quarantine::DatabaseError => e
@@ -103,24 +86,40 @@ class Quarantine
   end
 
   sig { void }
-  def upload_tests
-    return if @tests.empty? || @tests.values.count { |test| test.status == :quarantined } >= @options[:failsafe_limit]
+  def on_complete
+    quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
 
-    begin
-      timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
-      database.write_items(
-        @options[:test_statuses_table_name],
-        @tests.values.map { |item| item.to_hash.merge('updated_at' => timestamp) }
-      )
-    rescue Quarantine::DatabaseError => e
-      @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
+    if !@options[:record_tests]
+      log('Recording tests disabled; skipping')
+    elsif @tests.empty?
+      log('No tests found; skipping recording')
+    elsif quarantined_tests.count { |test| old_tests[test.id]&.status != :quarantined } >= @options[:failsafe_limit]
+      log('Number of quarantined tests above failsafe limit; skipping recording')
+    else
+      begin
+        timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
+        database.write_items(
+          @options[:test_statuses_table_name],
+          @tests.values.map { |item| item.to_hash.merge('updated_at' => timestamp) }
+        )
+      rescue Quarantine::DatabaseError => e
+        @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
+      end
     end
+
+    log(<<~MESSAGE)
+      \n[quarantine] Quarantined tests:
+        #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n  ")}
+
+      [quarantine] Database errors:
+        #{@database_failures.join("\n  ")}
+    MESSAGE
   end
 
   # Param: RSpec::Core::Example
   # Add the example to the internal tests list
   sig { params(example: T.untyped, status: Symbol, passed: T::Boolean).void }
-  def record_test(example, status, passed:)
+  def on_test(example, status, passed:)
     extra_attributes = @options[:extra_attributes] ? @options[:extra_attributes].call(example) : {}
 
     new_consecutive_passes = passed ? (@old_tests[example.id]&.consecutive_passes || 0) + 1 : 0
@@ -145,15 +144,8 @@ class Quarantine
     @old_tests[example.id]&.status == :quarantined
   end
 
-  sig { returns(String) }
-  def summary
-    quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
-    <<~MESSAGE
-      \n[quarantine] Quarantined tests:
-        #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n  ")}
-
-      [quarantine] Database errors:
-        #{@database_failures.join("\n  ")}
-    MESSAGE
+  sig { params(message: String).void }
+  def log(message)
+    @options[:log].call(message) if @options[:logging]
   end
 end

data/lib/quarantine/databases/google_sheets.rb

@@ -21,7 +21,7 @@ class Quarantine
       sig { override.params(table_name: String).returns(T::Enumerable[Item]) }
       def fetch_items(table_name)
         parse_rows(spreadsheet.worksheet_by_title(table_name))
-      rescue GoogleDrive::Error
+      rescue GoogleDrive::Error, Google::Apis::Error
         raise Quarantine::DatabaseError
       end
 
@@ -41,7 +41,12 @@ class Quarantine
         indexes = Hash[parsed_rows.each_with_index.map { |item, idx| [item['id'], idx] }]
 
         items.each do |item|
-          cells = headers.map { |header| item[header].to_s }
+          cells = headers.map do |header|
+            match = header.match(/^(extra_)?(.+)/)
+            extra, name = match[1..2]
+            value = extra ? item['extra_attributes'][name] : item[name]
+            value.to_s
+          end
           row_idx = indexes[item['id']]
           if row_idx
             # Overwrite existing row
@@ -56,7 +61,7 @@ class Quarantine
         # Insert any items whose IDs weren't found in existing rows at the end
         worksheet.insert_rows(parsed_rows.count + 2, new_rows)
         worksheet.save
-      rescue GoogleDrive::Error
+      rescue GoogleDrive::Error, Google::Apis::Error
         raise Quarantine::DatabaseError
       end
 
@@ -103,10 +108,12 @@ class Quarantine
         rows.map do |row|
           hash_row = Hash[headers.zip(row)]
           # TODO: use Google Sheets developer metadata to store type information
-          unless hash_row['id'].empty?
-            hash_row['extra_attributes'] = JSON.parse(hash_row['extra_attributes']) if hash_row['extra_attributes']
-            hash_row
-          end
+          next nil if hash_row['id'].empty?
+
+          extra_values, base_values = hash_row.partition { |k, _v| k.start_with?('extra_') }
+          base_hash = Hash[base_values]
+          base_hash['extra_attributes'] = Hash[extra_values]
+          base_hash
         end.compact
       end
     end

data/lib/quarantine/rspec_adapter.rb

@@ -1,19 +1,32 @@
 # typed: strict
 
+module RSpec
+  module Core
+    class Example
+      extend T::Sig
+
+      # The implementation of clear_exception in rspec-retry doesn't work
+      # for examples that use `it_behaves_like`, so we implement our own version that
+      # clear the exception field recursively.
+      sig { void }
+      def clear_exception!
+        @exception = T.let(nil, T.untyped)
+        T.unsafe(self).example.clear_exception! if defined?(example)
+      end
+    end
+  end
+end
+
 class Quarantine
   module RSpecAdapter
     extend T::Sig
 
-    # Purpose: create an instance of Quarantine which contains information
-    #          about the test suite (ie. quarantined tests) and binds RSpec configurations
-    #          and hooks onto the global RSpec class
     sig { void }
     def self.bind
-      bind_rspec_configurations
-      bind_fetch_test_statuses
-      bind_record_tests
-      bind_upload_tests
-      bind_logger
+      register_rspec_configurations
+      bind_on_start
+      bind_on_test
+      bind_on_complete
     end
 
     sig { returns(Quarantine) }
@@ -25,12 +38,15 @@ class Quarantine
         extra_attributes: RSpec.configuration.quarantine_extra_attributes,
         failsafe_limit: RSpec.configuration.quarantine_failsafe_limit,
         release_at_consecutive_passes: RSpec.configuration.quarantine_release_at_consecutive_passes,
+        logging: RSpec.configuration.quarantine_logging,
+        log: method(:log),
+        record_tests: RSpec.configuration.quarantine_record_tests
       )
     end
 
     # Purpose: binds rspec configuration variables
     sig { void }
-    def self.bind_rspec_configurations
+    def self.register_rspec_configurations
       ::RSpec.configure do |config|
         config.add_setting(:quarantine_database, default: { type: :dynamodb, region: 'us-west-1' })
         config.add_setting(:quarantine_test_statuses, { default: 'test_statuses' })
@@ -45,65 +61,72 @@ class Quarantine
 
     # Purpose: binds quarantine to fetch the test_statuses from dynamodb in the before suite
     sig { void }
-    def self.bind_fetch_test_statuses
+    def self.bind_on_start
       ::RSpec.configure do |config|
         config.before(:suite) do
-          Quarantine::RSpecAdapter.quarantine.fetch_test_statuses
+          Quarantine::RSpecAdapter.quarantine.on_start
         end
       end
     end
 
-    # Purpose: binds quarantine to record test statuses
-    sig { void }
-    def self.bind_record_tests
-      ::RSpec.configure do |config|
-        config.after(:each) do |example|
-          metadata = example.metadata
+    sig { params(example: RSpec::Core::Example).returns(T.nilable([Symbol, T::Boolean])) }
+    def self.final_status(example)
+      metadata = example.metadata
+
+      # The user may define their own after hook that marks an example as flaky in its metadata.
+      previously_quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
 
-          # optionally, the upstream RSpec configuration could define an after hook that marks an example as flaky in
-          # the example's metadata
-          quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
-          if example.exception
-            if metadata[:retry_attempts] + 1 == metadata[:retry]
-              # will record the failed test if it's final retry from the rspec-retry gem
-              if RSpec.configuration.skip_quarantined_tests && quarantined
-                example.clear_exception!
-                Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
-              else
-                Quarantine::RSpecAdapter.quarantine.record_test(example, :failing, passed: false)
-              end
-            end
-          elsif metadata[:retry_attempts] > 0
-            # will record the flaky test if it failed the first run but passed a subsequent run
-            Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
-          elsif quarantined
-            Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: true)
+      if example.exception
+        # The example failed _this try_.
+        if metadata[:retry_attempts] + 1 == metadata[:retry]
+          # The example failed all its retries - if it's already quarantined, keep it that way;
+          # otherwise, mark it as failing.
+          if RSpec.configuration.skip_quarantined_tests && previously_quarantined
+            return [:quarantined, false]
           else
-            Quarantine::RSpecAdapter.quarantine.record_test(example, :passing, passed: true)
+            return [:failing, false]
           end
         end
+        # The example failed, but it's not the final retry yet, so return nil.
+        return nil # rubocop:disable Style/RedundantReturn
+      elsif metadata[:retry_attempts] > 0
+        # The example passed this time, but failed one or more times before - the definition of a flaky test.
+        return [:quarantined, false] # rubocop:disable Style/RedundantReturn
+      elsif previously_quarantined
+        # The example passed the first time, but it's already marked quarantined, so keep it that way.
+        return [:quarantined, true] # rubocop:disable Style/RedundantReturn
+      else
+        return [:passing, true] # rubocop:disable Style/RedundantReturn
       end
     end
 
+    # Purpose: binds quarantine to record test statuses
     sig { void }
-    def self.bind_upload_tests
+    def self.bind_on_test
       ::RSpec.configure do |config|
-        config.after(:suite) do
-          Quarantine::RSpecAdapter.quarantine.upload_tests if RSpec.configuration.quarantine_record_tests
+        config.after(:each) do |example|
+          result = Quarantine::RSpecAdapter.final_status(example)
+          if result
+            status, passed = result
+            example.clear_exception! if status == :quarantined && !passed
+            Quarantine::RSpecAdapter.quarantine.on_test(example, status, passed: passed)
+          end
         end
       end
     end
 
-    # Purpose: binds quarantine logger to output test to RSpec formatter messages
     sig { void }
-    def self.bind_logger
+    def self.bind_on_complete
       ::RSpec.configure do |config|
         config.after(:suite) do
-          if RSpec.configuration.quarantine_logging
-            RSpec.configuration.reporter.message(Quarantine::RSpecAdapter.quarantine.summary)
-          end
+          Quarantine::RSpecAdapter.quarantine.on_complete
         end
       end
     end
+
+    sig { params(message: String).void }
+    def self.log(message)
+      RSpec.configuration.reporter.message(message)
+    end
   end
 end

data/lib/quarantine/version.rb

@@ -1,3 +1,3 @@
 class Quarantine
-  VERSION = '2.1.2'.freeze
+  VERSION = '2.2.2'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quarantine
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.2
 platform: ruby
 authors:
 - Flexport Engineering, Eric Zhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-14 00:00:00.000000000 Z
+date: 2021-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.0.2
 signing_key:
 specification_version: 4
 summary: Quarantine flaky RSpec tests