quarantine 2.0.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 47aa8e8df1db802259f0107a7a06c2d05162899b803ffd224da8a53246c1e105
4
- data.tar.gz: 8cb6c6c9c5b79666c10feaf38e0077eab27b57d494dc08e0c385925de1c89583
3
+ metadata.gz: c846ed3cf614eecefab5610bfb5d693444099fe90b6b429e4c697c16a014e21b
4
+ data.tar.gz: 5d17a24b4735f4c34af2d0902c0e3a7199750ce592694b5f9fe375928ada9d03
5
5
  SHA512:
6
- metadata.gz: e3f9ba5314c425861f18b7451a7f28f8ced4af50479ad880f0be263a9104ba3c6dd85bcada8a0c89d194e9487d02702c9fe4930caceebf835413891fc31d6a37
7
- data.tar.gz: 3177827c0ff075302fa8e62c0b3480728769a3cb6199cb8eb0e510237df9839d0efc10db0180d32016d1ced933d9d50f87336f5f2d6ff3f741622a5aed3364b2
6
+ metadata.gz: cb20ce19d6581b4836486176a080a6204a66f93d609e2b76a5d7bb91886b97e07466026fe45d0d7d4c16b8d2f072212a4ff982f42c40a55d68f42413276a92c4
7
+ data.tar.gz: bc093985a12b77f636d577b3fed3ab783f5fe206350a759a796097eee93976f42cb2fa199101a853f81d729aa109cce612f6deb4ba75748779281086e2591bca
data/README.md CHANGED
@@ -97,9 +97,20 @@ end
97
97
 
98
98
  Quarantine comes with built-in support for the following database types:
99
99
  - `:dynamodb`
100
+ - `:google_sheets`
100
101
 
101
102
  To use `:dynamodb`, be sure to add `gem 'aws-sdk-dynamodb', '~> 1', group: :test` to your `Gemfile`.
102
103
 
104
+ To use `:google_sheets`, be sure to add `gem 'google_drive', '~> 2', group: :test` to your `Gemfile`. Here's an example:
105
+
106
+ ```rb
107
+ config.quarantine_database = {
108
+ type: :google_sheets,
109
+ authorization: {type: :service_account_key, file: "service_account.json"}, # also accepts `type: :config`
110
+ spreadsheet: {type: :by_key, "1Jb5fC6wSuIMnP85tUR5knuZ4f5fuu4nMzQF6-0l-EXAMPLE"}, # also accepts `type: :by_title` and `type: :by_url`
111
+ }
112
+ ```
113
+
103
114
  To use a custom database that's not provided, subclass `Quarantine::Databases::Base` and pass an instance of your class as the `quarantine_database` setting:
104
115
 
105
116
  ```rb
data/lib/quarantine.rb CHANGED
@@ -7,23 +7,7 @@ require 'quarantine/rspec_adapter'
7
7
  require 'quarantine/test'
8
8
  require 'quarantine/databases/base'
9
9
  require 'quarantine/databases/dynamo_db'
10
-
11
- module RSpec
12
- module Core
13
- class Example
14
- extend T::Sig
15
-
16
- # The implementation of clear_exception in rspec-retry doesn't work
17
- # for examples that use `it_behaves_like`, so we implement our own version that
18
- # clear the exception field recursively.
19
- sig { void }
20
- def clear_exception!
21
- @exception = T.let(nil, T.untyped)
22
- T.unsafe(self).example.clear_exception! if defined?(example)
23
- end
24
- end
25
- end
26
- end
10
+ require 'quarantine/databases/google_sheets'
27
11
 
28
12
  class Quarantine
29
13
  extend T::Sig
@@ -55,6 +39,8 @@ class Quarantine
55
39
  case type
56
40
  when :dynamodb
57
41
  Quarantine::Databases::DynamoDB.new(database_options)
42
+ when :google_sheets
43
+ Quarantine::Databases::GoogleSheets.new(database_options)
58
44
  else
59
45
  raise Quarantine::UnsupportedDatabaseError.new("Quarantine does not support database type: #{type.inspect}")
60
46
  end
@@ -63,7 +49,7 @@ class Quarantine
63
49
 
64
50
  # Scans the test_statuses from the database and store their IDs in quarantined_ids
65
51
  sig { void }
66
- def fetch_test_statuses
52
+ def on_start
67
53
  begin
68
54
  test_statuses = database.fetch_items(@options[:test_statuses_table_name])
69
55
  rescue Quarantine::DatabaseError => e
@@ -100,24 +86,40 @@ class Quarantine
100
86
  end
101
87
 
102
88
  sig { void }
103
- def upload_tests
104
- return if @tests.empty? || @tests.values.count { |test| test.status == :quarantined } >= @options[:failsafe_limit]
89
+ def on_complete
90
+ quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
105
91
 
106
- begin
107
- timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
108
- database.write_items(
109
- @options[:test_statuses_table_name],
110
- @tests.values.map { |item| item.to_hash.merge(updated_at: timestamp) }
111
- )
112
- rescue Quarantine::DatabaseError => e
113
- @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
92
+ if !@options[:record_tests]
93
+ log('Recording tests disabled; skipping')
94
+ elsif @tests.empty?
95
+ log('No tests found; skipping recording')
96
+ elsif quarantined_tests.count { |test| old_tests[test.id]&.status != :quarantined } >= @options[:failsafe_limit]
97
+ log('Number of quarantined tests above failsafe limit; skipping recording')
98
+ else
99
+ begin
100
+ timestamp = Time.now.to_i / 1000 # Truncated millisecond from timestamp for reasons specific to Flexport
101
+ database.write_items(
102
+ @options[:test_statuses_table_name],
103
+ @tests.values.map { |item| item.to_hash.merge('updated_at' => timestamp) }
104
+ )
105
+ rescue Quarantine::DatabaseError => e
106
+ @database_failures << "#{e.cause&.class}: #{e.cause&.message}"
107
+ end
114
108
  end
109
+
110
+ log(<<~MESSAGE)
111
+ \n[quarantine] Quarantined tests:
112
+ #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n ")}
113
+
114
+ [quarantine] Database errors:
115
+ #{@database_failures.join("\n ")}
116
+ MESSAGE
115
117
  end
116
118
 
117
119
  # Param: RSpec::Core::Example
118
120
  # Add the example to the internal tests list
119
121
  sig { params(example: T.untyped, status: Symbol, passed: T::Boolean).void }
120
- def record_test(example, status, passed:)
122
+ def on_test(example, status, passed:)
121
123
  extra_attributes = @options[:extra_attributes] ? @options[:extra_attributes].call(example) : {}
122
124
 
123
125
  new_consecutive_passes = passed ? (@old_tests[example.id]&.consecutive_passes || 0) + 1 : 0
@@ -142,15 +144,8 @@ class Quarantine
142
144
  @old_tests[example.id]&.status == :quarantined
143
145
  end
144
146
 
145
- sig { returns(String) }
146
- def summary
147
- quarantined_tests = @tests.values.select { |test| test.status == :quarantined }.sort_by(&:id)
148
- <<~MESSAGE
149
- \n[quarantine] Quarantined tests:
150
- #{quarantined_tests.map { |test| "#{test.id} #{test.full_description}" }.join("\n ")}
151
-
152
- [quarantine] Database errors:
153
- #{@database_failures.join("\n ")}
154
- MESSAGE
147
+ sig { params(message: String).void }
148
+ def log(message)
149
+ @options[:log].call(message) if @options[:logging]
155
150
  end
156
151
  end
@@ -8,16 +8,7 @@ class Quarantine
8
8
 
9
9
  abstract!
10
10
 
11
- Item = T.type_alias do
12
- {
13
- 'id' => String,
14
- 'last_status' => String,
15
- 'consecutive_passes' => Integer,
16
- 'full_description' => String,
17
- 'location' => String,
18
- 'extra_attributes' => T.untyped
19
- }
20
- end
11
+ Item = T.type_alias { T::Hash[String, T.untyped] } # TODO: must have `id` key
21
12
 
22
13
  sig { abstract.params(table_name: String).returns(T::Enumerable[Item]) }
23
14
  def fetch_items(table_name); end
@@ -57,18 +57,6 @@ class Quarantine
57
57
  raise Quarantine::DatabaseError
58
58
  end
59
59
 
60
- sig { params(table_name: String, keys: T::Hash[T.untyped, T.untyped]).void }
61
- def delete_items(table_name, keys)
62
- @dynamodb.delete_item(
63
- table_name: table_name,
64
- key: {
65
- **keys
66
- }
67
- )
68
- rescue Aws::DynamoDB::Errors::ServiceError
69
- raise Quarantine::DatabaseError
70
- end
71
-
72
60
  sig do
73
61
  params(
74
62
  table_name: String,
@@ -0,0 +1,122 @@
1
+ # typed: strict
2
+
3
+ begin
4
+ require 'google_drive'
5
+ rescue LoadError
6
+ end
7
+ require 'quarantine/databases/base'
8
+
9
+ class Quarantine
10
+ module Databases
11
+ class GoogleSheets < Base
12
+ extend T::Sig
13
+
14
+ sig { params(options: T::Hash[T.untyped, T.untyped]).void }
15
+ def initialize(options)
16
+ super()
17
+
18
+ @options = options
19
+ end
20
+
21
+ sig { override.params(table_name: String).returns(T::Enumerable[Item]) }
22
+ def fetch_items(table_name)
23
+ parse_rows(spreadsheet.worksheet_by_title(table_name))
24
+ rescue GoogleDrive::Error, Google::Apis::Error
25
+ raise Quarantine::DatabaseError
26
+ end
27
+
28
+ sig do
29
+ override.params(
30
+ table_name: String,
31
+ items: T::Array[Item]
32
+ ).void
33
+ end
34
+ def write_items(table_name, items)
35
+ worksheet = spreadsheet.worksheet_by_title(table_name)
36
+ headers = worksheet.rows.first.reject(&:empty?)
37
+ new_rows = []
38
+
39
+ # Map existing ID to row index
40
+ parsed_rows = parse_rows(worksheet)
41
+ indexes = Hash[parsed_rows.each_with_index.map { |item, idx| [item['id'], idx] }]
42
+
43
+ items.each do |item|
44
+ cells = headers.map do |header|
45
+ match = header.match(/^(extra_)?(.+)/)
46
+ extra, name = match[1..]
47
+ puts "header: #{header}, extra: #{extra}, name: #{name}"
48
+ value = extra ? item['extra_attributes'][name.to_sym] : item[name]
49
+ value.to_s
50
+ end
51
+ row_idx = indexes[item['id']]
52
+ if row_idx
53
+ # Overwrite existing row
54
+ headers.each_with_index do |_header, col_idx|
55
+ worksheet[row_idx + 2, col_idx + 1] = cells[col_idx]
56
+ end
57
+ else
58
+ new_rows << cells
59
+ end
60
+ end
61
+
62
+ # Insert any items whose IDs weren't found in existing rows at the end
63
+ worksheet.insert_rows(parsed_rows.count + 2, new_rows)
64
+ worksheet.save
65
+ rescue GoogleDrive::Error, Google::Apis::Error
66
+ raise Quarantine::DatabaseError
67
+ end
68
+
69
+ private
70
+
71
+ sig { returns(GoogleDrive::Session) }
72
+ def session
73
+ @session = T.let(@session, T.nilable(GoogleDrive::Session))
74
+ @session ||= begin
75
+ authorization = @options[:authorization]
76
+ case authorization[:type]
77
+ when :service_account_key
78
+ GoogleDrive::Session.from_service_account_key(authorization[:file])
79
+ when :config
80
+ GoogleDrive::Session.from_config(authorization[:file])
81
+ else
82
+ raise "Invalid authorization type: #{authorization[:type]}"
83
+ end
84
+ end
85
+ end
86
+
87
+ sig { returns(GoogleDrive::Spreadsheet) }
88
+ def spreadsheet
89
+ @spreadsheet = T.let(@spreadsheet, T.nilable(GoogleDrive::Spreadsheet))
90
+ @spreadsheet ||= begin
91
+ spreadsheet = @options[:spreadsheet]
92
+ case spreadsheet[:type]
93
+ when :by_key
94
+ session.spreadsheet_by_key(spreadsheet[:key])
95
+ when :by_title
96
+ session.spreadsheet_by_title(spreadsheet[:title])
97
+ when :by_url
98
+ session.spreadsheet_by_url(spreadsheet[:url])
99
+ else
100
+ raise "Invalid spreadsheet type: #{spreadsheet[:type]}"
101
+ end
102
+ end
103
+ end
104
+
105
+ sig { params(worksheet: GoogleDrive::Worksheet).returns(T::Enumerable[Item]) }
106
+ def parse_rows(worksheet)
107
+ headers, *rows = worksheet.rows
108
+
109
+ rows.map do |row|
110
+ hash_row = Hash[headers.zip(row)]
111
+ # TODO: use Google Sheets developer metadata to store type information
112
+ unless hash_row['id'].empty?
113
+ extra_values, base_values = hash_row.partition{|k, v| k.start_with?('extra_')}
114
+ base_hash = Hash[base_values]
115
+ base_hash['extra_attributes'] = Hash[extra_values]
116
+ base_hash
117
+ end
118
+ end.compact
119
+ end
120
+ end
121
+ end
122
+ end
@@ -1,106 +1,132 @@
1
1
  # typed: strict
2
2
 
3
- module Quarantine::RSpecAdapter # rubocop:disable Style/ClassAndModuleChildren
4
- extend T::Sig
3
+ module RSpec
4
+ module Core
5
+ class Example
6
+ extend T::Sig
5
7
 
6
- # Purpose: create an instance of Quarantine which contains information
7
- # about the test suite (ie. quarantined tests) and binds RSpec configurations
8
- # and hooks onto the global RSpec class
9
- sig { void }
10
- def self.bind
11
- bind_rspec_configurations
12
- bind_fetch_test_statuses
13
- bind_record_tests
14
- bind_upload_tests
15
- bind_logger
8
+ # The implementation of clear_exception in rspec-retry doesn't work
9
+ # for examples that use `it_behaves_like`, so we implement our own version that
10
+ # clear the exception field recursively.
11
+ sig { void }
12
+ def clear_exception!
13
+ @exception = T.let(nil, T.untyped)
14
+ T.unsafe(self).example.clear_exception! if defined?(example)
15
+ end
16
+ end
16
17
  end
18
+ end
17
19
 
18
- sig { returns(Quarantine) }
19
- def self.quarantine
20
- @quarantine = T.let(@quarantine, T.nilable(Quarantine))
21
- @quarantine ||= Quarantine.new(
22
- database: RSpec.configuration.quarantine_database,
23
- test_statuses_table_name: RSpec.configuration.quarantine_test_statuses,
24
- extra_attributes: RSpec.configuration.quarantine_extra_attributes,
25
- failsafe_limit: RSpec.configuration.quarantine_failsafe_limit
26
- )
27
- end
20
+ class Quarantine
21
+ module RSpecAdapter
22
+ extend T::Sig
28
23
 
29
- # Purpose: binds rspec configuration variables
30
- sig { void }
31
- def self.bind_rspec_configurations
32
- ::RSpec.configure do |config|
33
- config.add_setting(:quarantine_database, default: { type: :dynamodb, region: 'us-west-1' })
34
- config.add_setting(:quarantine_test_statuses, { default: 'test_statuses' })
35
- config.add_setting(:skip_quarantined_tests, { default: true })
36
- config.add_setting(:quarantine_record_tests, { default: true })
37
- config.add_setting(:quarantine_logging, { default: true })
38
- config.add_setting(:quarantine_extra_attributes)
39
- config.add_setting(:quarantine_failsafe_limit, default: 10)
40
- config.add_setting(:quarantine_release_at_consecutive_passes)
24
+ sig { void }
25
+ def self.bind
26
+ register_rspec_configurations
27
+ bind_on_start
28
+ bind_on_test
29
+ bind_on_complete
41
30
  end
42
- end
43
31
 
44
- # Purpose: binds quarantine to fetch the test_statuses from dynamodb in the before suite
45
- sig { void }
46
- def self.bind_fetch_test_statuses
47
- ::RSpec.configure do |config|
48
- config.before(:suite) do
49
- Quarantine::RSpecAdapter.quarantine.fetch_test_statuses
32
+ sig { returns(Quarantine) }
33
+ def self.quarantine
34
+ @quarantine = T.let(@quarantine, T.nilable(Quarantine))
35
+ @quarantine ||= Quarantine.new(
36
+ database: RSpec.configuration.quarantine_database,
37
+ test_statuses_table_name: RSpec.configuration.quarantine_test_statuses,
38
+ extra_attributes: RSpec.configuration.quarantine_extra_attributes,
39
+ failsafe_limit: RSpec.configuration.quarantine_failsafe_limit,
40
+ release_at_consecutive_passes: RSpec.configuration.quarantine_release_at_consecutive_passes,
41
+ logging: RSpec.configuration.quarantine_logging,
42
+ log: method(:log),
43
+ record_tests: RSpec.configuration.quarantine_record_tests
44
+ )
45
+ end
46
+
47
+ # Purpose: binds rspec configuration variables
48
+ sig { void }
49
+ def self.register_rspec_configurations
50
+ ::RSpec.configure do |config|
51
+ config.add_setting(:quarantine_database, default: { type: :dynamodb, region: 'us-west-1' })
52
+ config.add_setting(:quarantine_test_statuses, { default: 'test_statuses' })
53
+ config.add_setting(:skip_quarantined_tests, { default: true })
54
+ config.add_setting(:quarantine_record_tests, { default: true })
55
+ config.add_setting(:quarantine_logging, { default: true })
56
+ config.add_setting(:quarantine_extra_attributes)
57
+ config.add_setting(:quarantine_failsafe_limit, default: 10)
58
+ config.add_setting(:quarantine_release_at_consecutive_passes)
50
59
  end
51
60
  end
52
- end
53
61
 
54
- # Purpose: binds quarantine to record test statuses
55
- sig { void }
56
- def self.bind_record_tests
57
- ::RSpec.configure do |config|
58
- config.after(:each) do |example|
59
- metadata = example.metadata
62
+ # Purpose: binds quarantine to fetch the test_statuses from dynamodb in the before suite
63
+ sig { void }
64
+ def self.bind_on_start
65
+ ::RSpec.configure do |config|
66
+ config.before(:suite) do
67
+ Quarantine::RSpecAdapter.quarantine.on_start
68
+ end
69
+ end
70
+ end
71
+
72
+ sig { params(example: RSpec::Core::Example).returns(T.nilable([Symbol, T::Boolean])) }
73
+ def self.final_status(example)
74
+ metadata = example.metadata
75
+
76
+ # The user may define their own after hook that marks an example as flaky in its metadata.
77
+ previously_quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
60
78
 
61
- # optionally, the upstream RSpec configuration could define an after hook that marks an example as flaky in
62
- # the example's metadata
63
- quarantined = Quarantine::RSpecAdapter.quarantine.test_quarantined?(example) || metadata[:flaky]
64
- if example.exception
65
- if metadata[:retry_attempts] + 1 == metadata[:retry]
66
- # will record the failed test if it's final retry from the rspec-retry gem
67
- if RSpec.configuration.skip_quarantined_tests && quarantined
68
- example.clear_exception!
69
- Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
70
- else
71
- Quarantine::RSpecAdapter.quarantine.record_test(example, :failing, passed: false)
72
- end
79
+ if example.exception
80
+ # The example failed _this try_.
81
+ if metadata[:retry_attempts] + 1 == metadata[:retry]
82
+ # The example failed all its retries - if it's already quarantined, keep it that way;
83
+ # otherwise, mark it as failing.
84
+ if RSpec.configuration.skip_quarantined_tests && previously_quarantined
85
+ return [:quarantined, false]
86
+ else
87
+ return [:failing, false]
73
88
  end
74
- elsif metadata[:retry_attempts] > 0
75
- # will record the flaky test if it failed the first run but passed a subsequent run
76
- Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: false)
77
- elsif quarantined
78
- Quarantine::RSpecAdapter.quarantine.record_test(example, :quarantined, passed: true)
79
- else
80
- Quarantine::RSpecAdapter.quarantine.record_test(example, :passing, passed: true)
81
89
  end
90
+ # The example failed, but it's not the final retry yet, so return nil.
91
+ return nil
92
+ elsif metadata[:retry_attempts] > 0
93
+ # The example passed this time, but failed one or more times before - the definition of a flaky test.
94
+ return [:quarantined, false]
95
+ elsif previously_quarantined
96
+ # The example passed the first time, but it's already marked quarantined, so keep it that way.
97
+ return [:quarantined, true]
98
+ else
99
+ return [:passing, true]
82
100
  end
83
101
  end
84
- end
85
102
 
86
- sig { void }
87
- def self.bind_upload_tests
88
- ::RSpec.configure do |config|
89
- config.after(:suite) do
90
- Quarantine::RSpecAdapter.quarantine.upload_tests if RSpec.configuration.quarantine_record_tests
103
+ # Purpose: binds quarantine to record test statuses
104
+ sig { void }
105
+ def self.bind_on_test
106
+ ::RSpec.configure do |config|
107
+ config.after(:each) do |example|
108
+ result = Quarantine::RSpecAdapter.final_status(example)
109
+ if result
110
+ status, passed = result
111
+ example.clear_exception! if status == :quarantined && !passed
112
+ Quarantine::RSpecAdapter.quarantine.on_test(example, status, passed: passed)
113
+ end
114
+ end
91
115
  end
92
116
  end
93
- end
94
117
 
95
- # Purpose: binds quarantine logger to output test to RSpec formatter messages
96
- sig { void }
97
- def self.bind_logger
98
- ::RSpec.configure do |config|
99
- config.after(:suite) do
100
- if RSpec.configuration.quarantine_logging
101
- RSpec.configuration.reporter.message(Quarantine::RSpecAdapter.quarantine.summary)
118
+ sig { void }
119
+ def self.bind_on_complete
120
+ ::RSpec.configure do |config|
121
+ config.after(:suite) do
122
+ Quarantine::RSpecAdapter.quarantine.on_complete
102
123
  end
103
124
  end
104
125
  end
126
+
127
+ sig { params(message: String).void }
128
+ def self.log(message)
129
+ RSpec.configuration.reporter.message(message)
130
+ end
105
131
  end
106
132
  end
@@ -1,3 +1,3 @@
1
1
  class Quarantine
2
- VERSION = '2.0.0'.freeze
2
+ VERSION = '2.2.0'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: quarantine
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.0
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Flexport Engineering, Eric Zhu
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-04-11 00:00:00.000000000 Z
11
+ date: 2021-04-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rspec
@@ -67,6 +67,7 @@ files:
67
67
  - lib/quarantine/cli.rb
68
68
  - lib/quarantine/databases/base.rb
69
69
  - lib/quarantine/databases/dynamo_db.rb
70
+ - lib/quarantine/databases/google_sheets.rb
70
71
  - lib/quarantine/error.rb
71
72
  - lib/quarantine/rspec_adapter.rb
72
73
  - lib/quarantine/test.rb