quarantine 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47aa8e8df1db802259f0107a7a06c2d05162899b803ffd224da8a53246c1e105
-  data.tar.gz: 8cb6c6c9c5b79666c10feaf38e0077eab27b57d494dc08e0c385925de1c89583
+  metadata.gz: c846ed3cf614eecefab5610bfb5d693444099fe90b6b429e4c697c16a014e21b
+  data.tar.gz: 5d17a24b4735f4c34af2d0902c0e3a7199750ce592694b5f9fe375928ada9d03
 SHA512:
-  metadata.gz: e3f9ba5314c425861f18b7451a7f28f8ced4af50479ad880f0be263a9104ba3c6dd85bcada8a0c89d194e9487d02702c9fe4930caceebf835413891fc31d6a37
-  data.tar.gz: 3177827c0ff075302fa8e62c0b3480728769a3cb6199cb8eb0e510237df9839d0efc10db0180d32016d1ced933d9d50f87336f5f2d6ff3f741622a5aed3364b2
+  metadata.gz: cb20ce19d6581b4836486176a080a6204a66f93d609e2b76a5d7bb91886b97e07466026fe45d0d7d4c16b8d2f072212a4ff982f42c40a55d68f42413276a92c4
+  data.tar.gz: bc093985a12b77f636d577b3fed3ab783f5fe206350a759a796097eee93976f42cb2fa199101a853f81d729aa109cce612f6deb4ba75748779281086e2591bca

data/README.md

@@ -97,9 +97,20 @@ end
 
 Quarantine comes with built-in support for the following database types:
 - `:dynamodb`
+- `:google_sheets`
 
 To use `:dynamodb`, be sure to add `gem 'aws-sdk-dynamodb', '~> 1', group: :test` to your `Gemfile`.
 
+To use `:google_sheets`, be sure to add `gem 'google_drive', '~> 2', group: :test` to your `Gemfile`. Here's an example:
+
+```rb
+config.quarantine_database = {
+  type: :google_sheets,
+  authorization: {type: :service_account_key, file: "service_account.json"}, # also accepts `type: :config`
+  spreadsheet: {type: :by_key, "1Jb5fC6wSuIMnP85tUR5knuZ4f5fuu4nMzQF6-0l-EXAMPLE"}, # also accepts `type: :by_title` and `type: :by_url`
+}
+```
+
 To use a custom database that's not provided, subclass `Quarantine::Databases::Base` and pass an instance of your class as the `quarantine_database` setting:
 
 ```rb

data/lib/quarantine.rb

@@ -7,23 +7,7 @@ require 'quarantine/rspec_adapter'
 require 'quarantine/test'
 require 'quarantine/databases/base'
 require 'quarantine/databases/dynamo_db'
-
-module RSpec
-  module Core
-    class Example
-      extend T::Sig
-
-      # The implementation of clear_exception in rspec-retry doesn't work
-      # for examples that use `it_behaves_like`, so we implement our own version that
-      # clear the exception field recursively.
-      sig { void }
-      def clear_exception!
-        @exception = T.let(nil, T.untyped)
-        T.unsafe(self).example.clear_exception! if defined?(example)
-      end
-    end
-  end
-end
+require 'quarantine/databases/google_sheets'
 
 class Quarantine
   extend T::Sig
@@ -55,6 +39,8 @@ class Quarantine
         case type
         when :dynamodb
           Quarantine::Databases::DynamoDB.new(database_options)
+        when :google_sheets
+          Quarantine::Databases::GoogleSheets.new(database_options)
         else
           raise Quarantine::UnsupportedDatabaseError.new("Quarantine does not support database type: #{type.inspect}")
         end
@@ -63,7 +49,7 @@ class Quarantine
 
   # Scans the test_statuses from the database and store their IDs in quarantined_ids
   sig { void }
-  def fetch_test_statuses
+  def on_start
     begin
       test_statuses = database.fetch_items(@options[:test_statuses_table_name])
     rescue Quarantine::DatabaseError => e
@@ -100,24 +86,40 @@ class Quarantine
   end
 
   sig { void }
-  def upload_tests
-    return if @tests.empty? || @tests.values.count { |test| test.status == :quarantined } >= @options[:failsafe_limit]
+  def on_complete
+    quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
 
-    begin
-      timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
-      database.write_items(
-        @options[:test_statuses_table_name],
-        @tests.values.map { |item| item.to_hash.merge(updated_at: timestamp) }
-      )
-    rescue Quarantine::DatabaseError => e
-      @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
+    if !@options[:record_tests]
+      log('Recording tests disabled; skipping')
+    elsif @tests.empty?
+      log('No tests found; skipping recording')
+    elsif quarantined_tests.count { |test| old_tests[test.id]&.status != :quarantined } >= @options[:failsafe_limit]
+      log('Number of quarantined tests above failsafe limit; skipping recording')
+    else
+      begin
+        timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
+        database.write_items(
+          @options[:test_statuses_table_name],
+          @tests.values.map { |item| item.to_hash.merge('updated_at' => timestamp) }
+        )
+      rescue Quarantine::DatabaseError => e
+        @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
+      end
     end
+
+    log(<<~MESSAGE)
+      \n[quarantine] Quarantined tests:
+        #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n  ")}
+
+      [quarantine] Database errors:
+        #{@database_failures.join("\n  ")}
+    MESSAGE
   end
 
   # Param: RSpec::Core::Example
   # Add the example to the internal tests list
   sig { params(example: T.untyped, status: Symbol, passed: T::Boolean).void }
-  def record_test(example, status, passed:)
+  def on_test(example, status, passed:)
     extra_attributes = @options[:extra_attributes] ? @options[:extra_attributes].call(example) : {}
 
     new_consecutive_passes = passed ? (@old_tests[example.id]&.consecutive_passes || 0) + 1 : 0
@@ -142,15 +144,8 @@ class Quarantine
     @old_tests[example.id]&.status == :quarantined
   end
 
-  sig { returns(String) }
-  def summary
-    quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
-    <<~MESSAGE
-      \n[quarantine] Quarantined tests:
-        #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n  ")}
-
-      [quarantine] Database errors:
-        #{@database_failures.join("\n  ")}
-    MESSAGE
+  sig { params(message: String).void }
+  def log(message)
+    @options[:log].call(message) if @options[:logging]
   end
 end

data/lib/quarantine/databases/base.rb

@@ -8,16 +8,7 @@ class Quarantine
 
       abstract!
 
-      Item = T.type_alias do
-        {
-          'id' => String,
-          'last_status' => String,
-          'consecutive_passes' => Integer,
-          'full_description' => String,
-          'location' => String,
-          'extra_attributes' => T.untyped
-        }
-      end
+      Item = T.type_alias { T::Hash[String, T.untyped] } # TODO: must have `id` key
 
       sig { abstract.params(table_name: String).returns(T::Enumerable[Item]) }
       def fetch_items(table_name); end

data/lib/quarantine/databases/dynamo_db.rb

@@ -57,18 +57,6 @@ class Quarantine
         raise Quarantine::DatabaseError
       end
 
-      sig { params(table_name: String, keys: T::Hash[T.untyped, T.untyped]).void }
-      def delete_items(table_name, keys)
-        @dynamodb.delete_item(
-          table_name: table_name,
-          key: {
-            **keys
-          }
-        )
-      rescue Aws::DynamoDB::Errors::ServiceError
-        raise Quarantine::DatabaseError
-      end
-
       sig do
         params(
           table_name: String,

data/lib/quarantine/databases/google_sheets.rb

@@ -0,0 +1,122 @@
+# typed: strict
+
+begin
+  require 'google_drive'
+rescue LoadError
+end
+require 'quarantine/databases/base'
+
+class Quarantine
+  module Databases
+    class GoogleSheets < Base
+      extend T::Sig
+
+      sig { params(options: T::Hash[T.untyped, T.untyped]).void }
+      def initialize(options)
+        super()
+
+        @options = options
+      end
+
+      sig { override.params(table_name: String).returns(T::Enumerable[Item]) }
+      def fetch_items(table_name)
+        parse_rows(spreadsheet.worksheet_by_title(table_name))
+      rescue GoogleDrive::Error, Google::Apis::Error
+        raise Quarantine::DatabaseError
+      end
+
+      sig do
+        override.params(
+          table_name: String,
+          items: T::Array[Item]
+        ).void
+      end
+      def write_items(table_name, items)
+        worksheet = spreadsheet.worksheet_by_title(table_name)
+        headers = worksheet.rows.first.reject(&:empty?)
+        new_rows = []
+
+        # Map existing ID to row index
+        parsed_rows = parse_rows(worksheet)
+        indexes = Hash[parsed_rows.each_with_index.map { |item, idx| [item['id'], idx] }]
+
+        items.each do |item|
+          cells = headers.map do |header|
+            match = header.match(/^(extra_)?(.+)/)
+            extra, name = match[1..]
+            puts "header: #{header}, extra: #{extra}, name: #{name}"
+            value = extra ? item['extra_attributes'][name.to_sym] : item[name]
+            value.to_s
+          end
+          row_idx = indexes[item['id']]
+          if row_idx
+            # Overwrite existing row
+            headers.each_with_index do |_header, col_idx|
+              worksheet[row_idx + 2, col_idx + 1] = cells[col_idx]
+            end
+          else
+            new_rows << cells
+          end
+        end
+
+        # Insert any items whose IDs weren't found in existing rows at the end
+        worksheet.insert_rows(parsed_rows.count + 2, new_rows)
+        worksheet.save
+      rescue GoogleDrive::Error, Google::Apis::Error
+        raise Quarantine::DatabaseError
+      end
+
+      private
+
+      sig { returns(GoogleDrive::Session) }
+      def session
+        @session = T.let(@session, T.nilable(GoogleDrive::Session))
+        @session ||= begin
+          authorization = @options[:authorization]
+          case authorization[:type]
+          when :service_account_key
+            GoogleDrive::Session.from_service_account_key(authorization[:file])
+          when :config
+            GoogleDrive::Session.from_config(authorization[:file])
+          else
+            raise "Invalid authorization type: #{authorization[:type]}"
+          end
+        end
+      end
+
+      sig { returns(GoogleDrive::Spreadsheet) }
+      def spreadsheet
+        @spreadsheet = T.let(@spreadsheet, T.nilable(GoogleDrive::Spreadsheet))
+        @spreadsheet ||= begin
+          spreadsheet = @options[:spreadsheet]
+          case spreadsheet[:type]
+          when :by_key
+            session.spreadsheet_by_key(spreadsheet[:key])
+          when :by_title
+            session.spreadsheet_by_title(spreadsheet[:title])
+          when :by_url
+            session.spreadsheet_by_url(spreadsheet[:url])
+          else
+            raise "Invalid spreadsheet type: #{spreadsheet[:type]}"
+          end
+        end
+      end
+
+      sig { params(worksheet: GoogleDrive::Worksheet).returns(T::Enumerable[Item]) }
+      def parse_rows(worksheet)
+        headers, *rows = worksheet.rows
+
+        rows.map do |row|
+          hash_row = Hash[headers.zip(row)]
+          # TODO: use Google Sheets developer metadata to store type information
+          unless hash_row['id'].empty?
+            extra_values, base_values = hash_row.partition{|k, v| k.start_with?('extra_')}
+            base_hash = Hash[base_values]
+            base_hash['extra_attributes'] = Hash[extra_values]
+            base_hash
+          end
+        end.compact
+      end
+    end
+  end
+end

data/lib/quarantine/rspec_adapter.rb

@@ -1,106 +1,132 @@
 # typed: strict
 
-module Quarantine::RSpecAdapter # rubocop:disable Style/ClassAndModuleChildren
-  extend T::Sig
+module RSpec
+  module Core
+    class Example
+      extend T::Sig
 
-  # Purpose: create an instance of Quarantine which contains information
-  #          about the test suite (ie. quarantined tests) and binds RSpec configurations
-  #          and hooks onto the global RSpec class
-  sig { void }
-  def self.bind
-    bind_rspec_configurations
-    bind_fetch_test_statuses
-    bind_record_tests
-    bind_upload_tests
-    bind_logger
+      # The implementation of clear_exception in rspec-retry doesn't work
+      # for examples that use `it_behaves_like`, so we implement our own version that
+      # clear the exception field recursively.
+      sig { void }
+      def clear_exception!
+        @exception = T.let(nil, T.untyped)
+        T.unsafe(self).example.clear_exception! if defined?(example)
+      end
+    end
   end
+end
 
-  sig { returns(Quarantine) }
-  def self.quarantine
-    @quarantine = T.let(@quarantine, T.nilable(Quarantine))
-    @quarantine ||= Quarantine.new(
-      database: RSpec.configuration.quarantine_database,
-      test_statuses_table_name: RSpec.configuration.quarantine_test_statuses,
-      extra_attributes: RSpec.configuration.quarantine_extra_attributes,
-      failsafe_limit: RSpec.configuration.quarantine_failsafe_limit
-    )
-  end
+class Quarantine
+  module RSpecAdapter
+    extend T::Sig
 
-  # Purpose: binds rspec configuration variables
-  sig { void }
-  def self.bind_rspec_configurations
-    ::RSpec.configure do |config|
-      config.add_setting(:quarantine_database, default: { type: :dynamodb, region: 'us-west-1' })
-      config.add_setting(:quarantine_test_statuses, { default: 'test_statuses' })
-      config.add_setting(:skip_quarantined_tests, { default: true })
-      config.add_setting(:quarantine_record_tests, { default: true })
-      config.add_setting(:quarantine_logging, { default: true })
-      config.add_setting(:quarantine_extra_attributes)
-      config.add_setting(:quarantine_failsafe_limit, default: 10)
-      config.add_setting(:quarantine_release_at_consecutive_passes)
+    sig { void }
+    def self.bind
+      register_rspec_configurations
+      bind_on_start
+      bind_on_test
+      bind_on_complete
     end
-  end
 
-  # Purpose: binds quarantine to fetch the test_statuses from dynamodb in the before suite
-  sig { void }
-  def self.bind_fetch_test_statuses
-    ::RSpec.configure do |config|
-      config.before(:suite) do
-        Quarantine::RSpecAdapter.quarantine.fetch_test_statuses
+    sig { returns(Quarantine) }
+    def self.quarantine
+      @quarantine = T.let(@quarantine, T.nilable(Quarantine))
+      @quarantine ||= Quarantine.new(
+        database: RSpec.configuration.quarantine_database,
+        test_statuses_table_name: RSpec.configuration.quarantine_test_statuses,
+        extra_attributes: RSpec.configuration.quarantine_extra_attributes,
+        failsafe_limit: RSpec.configuration.quarantine_failsafe_limit,
+        release_at_consecutive_passes: RSpec.configuration.quarantine_release_at_consecutive_passes,
+        logging: RSpec.configuration.quarantine_logging,
+        log: method(:log),
+        record_tests: RSpec.configuration.quarantine_record_tests
+      )
+    end
+
+    # Purpose: binds rspec configuration variables
+    sig { void }
+    def self.register_rspec_configurations
+      ::RSpec.configure do |config|
+        config.add_setting(:quarantine_database, default: { type: :dynamodb, region: 'us-west-1' })
+        config.add_setting(:quarantine_test_statuses, { default: 'test_statuses' })
+        config.add_setting(:skip_quarantined_tests, { default: true })
+        config.add_setting(:quarantine_record_tests, { default: true })
+        config.add_setting(:quarantine_logging, { default: true })
+        config.add_setting(:quarantine_extra_attributes)
+        config.add_setting(:quarantine_failsafe_limit, default: 10)
+        config.add_setting(:quarantine_release_at_consecutive_passes)
       end
     end
-  end
 
-  # Purpose: binds quarantine to record test statuses
-  sig { void }
-  def self.bind_record_tests
-    ::RSpec.configure do |config|
-      config.after(:each) do |example|
-        metadata = example.metadata
+    # Purpose: binds quarantine to fetch the test_statuses from dynamodb in the before suite
+    sig { void }
+    def self.bind_on_start
+      ::RSpec.configure do |config|
+        config.before(:suite) do
+          Quarantine::RSpecAdapter.quarantine.on_start
+        end
+      end
+    end
+
+    sig { params(example: RSpec::Core::Example).returns(T.nilable([Symbol, T::Boolean])) }
+    def self.final_status(example)
+      metadata = example.metadata
+
+      # The user may define their own after hook that marks an example as flaky in its metadata.
+      previously_quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
 
-        # optionally, the upstream RSpec configuration could define an after hook that marks an example as flaky in
-        # the example's metadata
-        quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
-        if example.exception
-          if metadata[:retry_attempts] + 1 == metadata[:retry]
-            # will record the failed test if it's final retry from the rspec-retry gem
-            if RSpec.configuration.skip_quarantined_tests && quarantined
-              example.clear_exception!
-              Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
-            else
-              Quarantine::RSpecAdapter.quarantine.record_test(example, :failing, passed: false)
-            end
+      if example.exception
+        # The example failed _this try_.
+        if metadata[:retry_attempts] + 1 == metadata[:retry]
+          # The example failed all its retries - if it's already quarantined, keep it that way;
+          # otherwise, mark it as failing.
+          if RSpec.configuration.skip_quarantined_tests && previously_quarantined
+            return [:quarantined, false]
+          else
+            return [:failing, false]
           end
-        elsif metadata[:retry_attempts] > 0
-          # will record the flaky test if it failed the first run but passed a subsequent run
-          Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
-        elsif quarantined
-          Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: true)
-        else
-          Quarantine::RSpecAdapter.quarantine.record_test(example, :passing, passed: true)
         end
+        # The example failed, but it's not the final retry yet, so return nil.
+        return nil
+      elsif metadata[:retry_attempts] > 0
+        # The example passed this time, but failed one or more times before - the definition of a flaky test.
+        return [:quarantined, false]
+      elsif previously_quarantined
+        # The example passed the first time, but it's already marked quarantined, so keep it that way.
+        return [:quarantined, true]
+      else
+        return [:passing, true]
       end
     end
-  end
 
-  sig { void }
-  def self.bind_upload_tests
-    ::RSpec.configure do |config|
-      config.after(:suite) do
-        Quarantine::RSpecAdapter.quarantine.upload_tests if RSpec.configuration.quarantine_record_tests
+    # Purpose: binds quarantine to record test statuses
+    sig { void }
+    def self.bind_on_test
+      ::RSpec.configure do |config|
+        config.after(:each) do |example|
+          result = Quarantine::RSpecAdapter.final_status(example)
+          if result
+            status, passed = result
+            example.clear_exception! if status == :quarantined && !passed
+            Quarantine::RSpecAdapter.quarantine.on_test(example, status, passed: passed)
+          end
+        end
       end
     end
-  end
 
-  # Purpose: binds quarantine logger to output test to RSpec formatter messages
-  sig { void }
-  def self.bind_logger
-    ::RSpec.configure do |config|
-      config.after(:suite) do
-        if RSpec.configuration.quarantine_logging
-          RSpec.configuration.reporter.message(Quarantine::RSpecAdapter.quarantine.summary)
+    sig { void }
+    def self.bind_on_complete
+      ::RSpec.configure do |config|
+        config.after(:suite) do
+          Quarantine::RSpecAdapter.quarantine.on_complete
         end
       end
     end
+
+    sig { params(message: String).void }
+    def self.log(message)
+      RSpec.configuration.reporter.message(message)
+    end
   end
 end

data/lib/quarantine/version.rb

@@ -1,3 +1,3 @@
 class Quarantine
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.2.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quarantine
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Flexport Engineering, Eric Zhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-11 00:00:00.000000000 Z
+date: 2021-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -67,6 +67,7 @@ files:
 - lib/quarantine/cli.rb
 - lib/quarantine/databases/base.rb
 - lib/quarantine/databases/dynamo_db.rb
+- lib/quarantine/databases/google_sheets.rb
 - lib/quarantine/error.rb
 - lib/quarantine/rspec_adapter.rb
 - lib/quarantine/test.rb