quarantine 2.1.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffb721a4514eda413ea4c096913b5d0c3a672b23fe6aefee44393062f202e453
-  data.tar.gz: eec03b0ec1533643ae86245e76a69b51fdbf871e6e621293fa542b1b252fc123
+  metadata.gz: 31ffed67bd1774918fd52c8bfe309398d43b8c218175f2a414725382df680ee7
+  data.tar.gz: aca3eab589422f555224ce2638e979aff7f6fc2f17d4b33ac91e2f0177d4fb9b
 SHA512:
-  metadata.gz: 67f36af412a0f1b23aaa018402e81b087f9595d3257726aac3b4e6472f232a498be52d683fd6b0216f652ad3f2b702d6a5f582de70a62a8fb25fc29ec5a21df1
-  data.tar.gz: 3aabf31b49b00cded2af25bac1b8d9f3c1c1bb89a202689a87ded5e23369fb6e4d32b82f664d3c4d3b1f314cae02f34dc80ca0a0380e6c034466e7ae8ec15300
+  metadata.gz: '0894a8c2561b741d3f9466d28e8424cde69ece0b7f228d700b485f748eb26531bd81ac549493a91641010ceb6b34eb1d7abbaf49475f4874d6f67db297d522ec'
+  data.tar.gz: 281ba613e8806b8d3f1e4318a65492435d971e0e89138cb87c44de2c7709a983672930e00cdfddb46bf245da4aa2e52804b2cf47d908fe9f0dd62eae3e7ac5fd

data/lib/quarantine.rb

@@ -9,23 +9,6 @@ require 'quarantine/databases/base'
 require 'quarantine/databases/dynamo_db'
 require 'quarantine/databases/google_sheets'
 
-module RSpec
-  module Core
-    class Example
-      extend T::Sig
-
-      # The implementation of clear_exception in rspec-retry doesn't work
-      # for examples that use `it_behaves_like`, so we implement our own version that
-      # clear the exception field recursively.
-      sig { void }
-      def clear_exception!
-        @exception = T.let(nil, T.untyped)
-        T.unsafe(self).example.clear_exception! if defined?(example)
-      end
-    end
-  end
-end
-
 class Quarantine
   extend T::Sig
 
@@ -66,7 +49,7 @@ class Quarantine
 
   # Scans the test_statuses from the database and store their IDs in quarantined_ids
   sig { void }
-  def fetch_test_statuses
+  def on_start
     begin
       test_statuses = database.fetch_items(@options[:test_statuses_table_name])
     rescue Quarantine::DatabaseError => e
@@ -103,24 +86,40 @@ class Quarantine
   end
 
   sig { void }
-  def upload_tests
-    return if @tests.empty? || @tests.values.count { |test| test.status == :quarantined } >= @options[:failsafe_limit]
+  def on_complete
+    quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
 
-    begin
-      timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
-      database.write_items(
-        @options[:test_statuses_table_name],
-        @tests.values.map { |item| item.to_hash.merge('updated_at' => timestamp) }
-      )
-    rescue Quarantine::DatabaseError => e
-      @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
+    if !@options[:record_tests]
+      log('Recording tests disabled; skipping')
+    elsif @tests.empty?
+      log('No tests found; skipping recording')
+    elsif quarantined_tests.count { |test| old_tests[test.id]&.status != :quarantined } >= @options[:failsafe_limit]
+      log('Number of quarantined tests above failsafe limit; skipping recording')
+    else
+      begin
+        timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
+        database.write_items(
+          @options[:test_statuses_table_name],
+          @tests.values.map { |item| item.to_hash.merge('updated_at' => timestamp) }
+        )
+      rescue Quarantine::DatabaseError => e
+        @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
+      end
     end
+
+    log(<<~MESSAGE)
+      \n[quarantine] Quarantined tests:
+        #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n  ")}
+
+      [quarantine] Database errors:
+        #{@database_failures.join("\n  ")}
+    MESSAGE
   end
 
   # Param: RSpec::Core::Example
   # Add the example to the internal tests list
   sig { params(example: T.untyped, status: Symbol, passed: T::Boolean).void }
-  def record_test(example, status, passed:)
+  def on_test(example, status, passed:)
     extra_attributes = @options[:extra_attributes] ? @options[:extra_attributes].call(example) : {}
 
     new_consecutive_passes = passed ? (@old_tests[example.id]&.consecutive_passes || 0) + 1 : 0
@@ -145,15 +144,8 @@ class Quarantine
     @old_tests[example.id]&.status == :quarantined
   end
 
-  sig { returns(String) }
-  def summary
-    quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
-    <<~MESSAGE
-      \n[quarantine] Quarantined tests:
-        #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n  ")}
-
-      [quarantine] Database errors:
-        #{@database_failures.join("\n  ")}
-    MESSAGE
+  sig { params(message: String).void }
+  def log(message)
+    @options[:log].call(message) if @options[:logging]
   end
 end

data/lib/quarantine/databases/google_sheets.rb

@@ -21,6 +21,8 @@ class Quarantine
       sig { override.params(table_name: String).returns(T::Enumerable[Item]) }
       def fetch_items(table_name)
         parse_rows(spreadsheet.worksheet_by_title(table_name))
+      rescue GoogleDrive::Error, Google::Apis::Error
+        raise Quarantine::DatabaseError
       end
 
       sig do
@@ -35,10 +37,16 @@ class Quarantine
         new_rows = []
 
         # Map existing ID to row index
-        indexes = Hash[parse_rows(worksheet).each_with_index.map { |item, idx| [item['id'], idx] }]
+        parsed_rows = parse_rows(worksheet)
+        indexes = Hash[parsed_rows.each_with_index.map { |item, idx| [item['id'], idx] }]
 
         items.each do |item|
-          cells = headers.map { |header| item[header].to_s }
+          cells = headers.map do |header|
+            match = header.match(/^(extra_)?(.+)/)
+            extra, name = match[1..2]
+            value = extra ? item['extra_attributes'][name] : item[name]
+            value.to_s
+          end
           row_idx = indexes[item['id']]
           if row_idx
             # Overwrite existing row
@@ -51,8 +59,10 @@ class Quarantine
         end
 
         # Insert any items whose IDs weren't found in existing rows at the end
-        worksheet.insert_rows(worksheet.rows.count + 1, new_rows)
+        worksheet.insert_rows(parsed_rows.count + 2, new_rows)
         worksheet.save
+      rescue GoogleDrive::Error, Google::Apis::Error
+        raise Quarantine::DatabaseError
       end
 
       private
@@ -98,9 +108,13 @@ class Quarantine
         rows.map do |row|
           hash_row = Hash[headers.zip(row)]
           # TODO: use Google Sheets developer metadata to store type information
-          hash_row['extra_attributes'] = JSON.parse(hash_row['extra_attributes']) if hash_row['extra_attributes']
-          hash_row
-        end
+          next nil if hash_row['id'].empty?
+
+          extra_values, base_values = hash_row.partition { |k, _v| k.start_with?('extra_') }
+          base_hash = Hash[base_values]
+          base_hash['extra_attributes'] = Hash[extra_values]
+          base_hash
+        end.compact
       end
     end
   end

data/lib/quarantine/rspec_adapter.rb

@@ -1,19 +1,32 @@
 # typed: strict
 
+module RSpec
+  module Core
+    class Example
+      extend T::Sig
+
+      # The implementation of clear_exception in rspec-retry doesn't work
+      # for examples that use `it_behaves_like`, so we implement our own version that
+      # clear the exception field recursively.
+      sig { void }
+      def clear_exception!
+        @exception = T.let(nil, T.untyped)
+        T.unsafe(self).example.clear_exception! if defined?(example)
+      end
+    end
+  end
+end
+
 class Quarantine
   module RSpecAdapter
     extend T::Sig
 
-    # Purpose: create an instance of Quarantine which contains information
-    #          about the test suite (ie. quarantined tests) and binds RSpec configurations
-    #          and hooks onto the global RSpec class
     sig { void }
     def self.bind
-      bind_rspec_configurations
-      bind_fetch_test_statuses
-      bind_record_tests
-      bind_upload_tests
-      bind_logger
+      register_rspec_configurations
+      bind_on_start
+      bind_on_test
+      bind_on_complete
     end
 
     sig { returns(Quarantine) }
@@ -23,13 +36,17 @@ class Quarantine
         database: RSpec.configuration.quarantine_database,
         test_statuses_table_name: RSpec.configuration.quarantine_test_statuses,
         extra_attributes: RSpec.configuration.quarantine_extra_attributes,
-        failsafe_limit: RSpec.configuration.quarantine_failsafe_limit
+        failsafe_limit: RSpec.configuration.quarantine_failsafe_limit,
+        release_at_consecutive_passes: RSpec.configuration.quarantine_release_at_consecutive_passes,
+        logging: RSpec.configuration.quarantine_logging,
+        log: method(:log),
+        record_tests: RSpec.configuration.quarantine_record_tests
       )
     end
 
     # Purpose: binds rspec configuration variables
     sig { void }
-    def self.bind_rspec_configurations
+    def self.register_rspec_configurations
       ::RSpec.configure do |config|
         config.add_setting(:quarantine_database, default: { type: :dynamodb, region: 'us-west-1' })
         config.add_setting(:quarantine_test_statuses, { default: 'test_statuses' })
@@ -44,65 +61,72 @@ class Quarantine
 
     # Purpose: binds quarantine to fetch the test_statuses from dynamodb in the before suite
     sig { void }
-    def self.bind_fetch_test_statuses
+    def self.bind_on_start
       ::RSpec.configure do |config|
         config.before(:suite) do
-          Quarantine::RSpecAdapter.quarantine.fetch_test_statuses
+          Quarantine::RSpecAdapter.quarantine.on_start
         end
       end
     end
 
-    # Purpose: binds quarantine to record test statuses
-    sig { void }
-    def self.bind_record_tests
-      ::RSpec.configure do |config|
-        config.after(:each) do |example|
-          metadata = example.metadata
+    sig { params(example: RSpec::Core::Example).returns(T.nilable([Symbol, T::Boolean])) }
+    def self.final_status(example)
+      metadata = example.metadata
+
+      # The user may define their own after hook that marks an example as flaky in its metadata.
+      previously_quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
 
-          # optionally, the upstream RSpec configuration could define an after hook that marks an example as flaky in
-          # the example's metadata
-          quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
-          if example.exception
-            if metadata[:retry_attempts] + 1 == metadata[:retry]
-              # will record the failed test if it's final retry from the rspec-retry gem
-              if RSpec.configuration.skip_quarantined_tests && quarantined
-                example.clear_exception!
-                Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
-              else
-                Quarantine::RSpecAdapter.quarantine.record_test(example, :failing, passed: false)
-              end
-            end
-          elsif metadata[:retry_attempts] > 0
-            # will record the flaky test if it failed the first run but passed a subsequent run
-            Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
-          elsif quarantined
-            Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: true)
+      if example.exception
+        # The example failed _this try_.
+        if metadata[:retry_attempts] + 1 == metadata[:retry]
+          # The example failed all its retries - if it's already quarantined, keep it that way;
+          # otherwise, mark it as failing.
+          if RSpec.configuration.skip_quarantined_tests && previously_quarantined
+            return [:quarantined, false]
           else
-            Quarantine::RSpecAdapter.quarantine.record_test(example, :passing, passed: true)
+            return [:failing, false]
           end
         end
+        # The example failed, but it's not the final retry yet, so return nil.
+        return nil # rubocop:disable Style/RedundantReturn
+      elsif metadata[:retry_attempts] > 0
+        # The example passed this time, but failed one or more times before - the definition of a flaky test.
+        return [:quarantined, false] # rubocop:disable Style/RedundantReturn
+      elsif previously_quarantined
+        # The example passed the first time, but it's already marked quarantined, so keep it that way.
+        return [:quarantined, true] # rubocop:disable Style/RedundantReturn
+      else
+        return [:passing, true] # rubocop:disable Style/RedundantReturn
       end
     end
 
+    # Purpose: binds quarantine to record test statuses
     sig { void }
-    def self.bind_upload_tests
+    def self.bind_on_test
       ::RSpec.configure do |config|
-        config.after(:suite) do
-          Quarantine::RSpecAdapter.quarantine.upload_tests if RSpec.configuration.quarantine_record_tests
+        config.after(:each) do |example|
+          result = Quarantine::RSpecAdapter.final_status(example)
+          if result
+            status, passed = result
+            example.clear_exception! if status == :quarantined && !passed
+            Quarantine::RSpecAdapter.quarantine.on_test(example, status, passed: passed)
+          end
         end
       end
     end
 
-    # Purpose: binds quarantine logger to output test to RSpec formatter messages
     sig { void }
-    def self.bind_logger
+    def self.bind_on_complete
       ::RSpec.configure do |config|
         config.after(:suite) do
-          if RSpec.configuration.quarantine_logging
-            RSpec.configuration.reporter.message(Quarantine::RSpecAdapter.quarantine.summary)
-          end
+          Quarantine::RSpecAdapter.quarantine.on_complete
         end
       end
     end
+
+    sig { params(message: String).void }
+    def self.log(message)
+      RSpec.configuration.reporter.message(message)
+    end
   end
 end

data/lib/quarantine/version.rb

@@ -1,3 +1,3 @@
 class Quarantine
-  VERSION = '2.1.0'.freeze
+  VERSION = '2.2.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quarantine
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.1
 platform: ruby
 authors:
 - Flexport Engineering, Eric Zhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-14 00:00:00.000000000 Z
+date: 2021-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec