rspec_trunk_flaky_tests 0.7.2-x86_64-darwin → 0.7.5-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbabae40fb4be2b45660632a6694991e62f0219592e87ae49b321db2d11bd9dc
-  data.tar.gz: 41824dd72b9a722838a702c4f97dcbf760a3ad0c8152dd85653079786111c330
+  metadata.gz: 86574eb1db68bf0f06f4b2e03c072b746a74e2da8cfe324af79ec3e6adc3bd67
+  data.tar.gz: 7e15460fbd3f2ec674df1b646f1ae751bfb86048819ab11f29c9b38d7bdbfdca
 SHA512:
-  metadata.gz: ca43212ff1653d7babf741919a356a43963e95a10ecbd269f1297e9b11761276cea203b12d54882a4ddba9b3a8f611d2cb45a92335cdeb75786ac3b033172b1a
-  data.tar.gz: 007cec47471a87c0b8cb55785ac8042287d6231f72265efb3a0fb5c848af61e3ea8a402fada4674e88d509e5558324127d3d5cbee9232a88d53629fe12976b24
+  metadata.gz: 451200763b498f417460f84bcc9daf80e8b69f6c996e2b59abc0b1bdcebba780cd395ef19b3966b4092e08cf17c8268ec3ee37d0ff0e04101b558285fb55dc24
+  data.tar.gz: 887b4af52a1814332fd9f2d99771d5d66a2d8ee04dce1898deac9d7fe2896827948b38c6344cd44f8655c41d80c6bb88d21180c20ca4be75c00e8aa5552bc6de

data/lib/trunk_spec_helper.rb

@@ -4,10 +4,47 @@ require 'rspec/core'
 require 'time'
 require 'context_ruby'
 
+class String
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def yellow
+    colorize(33)
+  end
+end
+
 def escape(str)
   str.dump[1..-2]
 end
 
+def description_generated?(example)
+  auto_generated_exp = /^\s?is expected to eq .*$/
+  full_description = example.full_description
+  parent_description = example.example_group.description
+  checked_description = full_description.sub(parent_description, '')
+  !auto_generated_exp.match(checked_description).nil? || full_description.empty?
+end
+
+def generate_id(example)
+  return "trunk:#{example.id}-#{example.location}" if description_generated?(example)
+end
+
+def trunk_disabled
+  ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' || ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil?
+end
+
+# we want to cache the test report so we can add to it as we go and reduce the number of API calls
+$test_report = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}")
+
 module RSpec
   module Core
     # Example is the class that represents a test case
@@ -17,16 +54,25 @@ module RSpec
       # RSpec uses the existance of an exception to determine if the test failed
       # We need to override this to allow us to capture the exception and then
       # decide if we want to fail the test or not
-      # trunk-ignore(rubocop/Naming/AccessorMethodName)
+      # trunk-ignore(rubocop/Naming/AccessorMethodName,rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
       def set_exception(exception)
-        # TODO: this is where we'll need to override the result once the logic is ready
-        # trunk-ignore(rubocop/Lint/LiteralAsCondition)
-        if true
-          set_exception_core(exception)
-        else
+        return set_exception_core(exception) if trunk_disabled
+
+        id = generate_id(self)
+        name = full_description
+        parent_name = example_group.metadata[:description]
+        parent_name = parent_name.empty? ? 'rspec' : parent_name
+        file = escape(metadata[:file_path])
+        classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
+        puts "Test failed, checking if it can be quarantined: `#{location}`".yellow
+        if $test_report.is_quarantined(id, name, parent_name, classname, file)
           # monitor the override in the metadata
           metadata[:quarantined_exception] = exception
+          puts "Test is quarantined, overriding exception: #{exception}".green
           nil
+        else
+          puts 'Test is not quarantined, continuing'.red
+          set_exception_core(exception)
         end
       end
 
@@ -45,7 +91,7 @@ module RSpec
   class Trunk
     def self.setup
       RSpec.configure do |config|
-        if ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true'
+        if trunk_disabled
           config.around(:each, &:run)
         else
           config.around(:each, &:run_with_trunk)
@@ -70,10 +116,6 @@ module RSpec
       else
         @example.metadata[:attempt_number] = 0
       end
-
-      # add the test to the report
-      # return the report
-      @testreport
     end
   end
 end
@@ -82,7 +124,7 @@ end
 # it generates and submits the final test reports
 class TrunkAnalyticsListener
   def initialize
-    @testreport = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}")
+    @testreport = $test_report
   end
 
   def example_finished(notification)
@@ -90,34 +132,18 @@ class TrunkAnalyticsListener
   end
 
   def close(_notification)
-    @testreport.publish
-  end
-
-  def description_generated?(example)
-    auto_generated_exp = /^\s?is expected to eq .*$/
-    full_description = example.full_description
-    parent_description = example.example_group.description
-    checked_description = full_description.sub(parent_description, '')
-    auto_generated_exp.match(checked_description) != nil
-  end
-
-  def generate_id(example)
-    if description_generated?(example)
-      # trunk-ignore(rubocop/Style/SoleNestedConditional)
-      return "trunk:#{example.id}-#{example.location}" if description_generated?(example)
+    res = @testreport.publish
+    if res
+      puts 'Flaky tests report upload complete'.green
+    else
+      puts 'Failed to publish flaky tests report'.red
     end
-    nil
   end
 
   # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength,rubocop/Metrics/CyclomaticComplexity)
   def add_test_case(example)
-    if example.exception
-      failure_message = example.exception.message
-      # failure details is far more robust than the message, but noiser
-      # if example.exception.backtrace
-      # failure_details = example.exception.backtrace.join('\n')
-      # end
-    end
+    failure_message = example.exception.to_s if example.exception
+    failure_message = example.metadata[:quarantined_exception].to_s if example.metadata[:quarantined_exception]
     # TODO: should we use concatenated string or alias when auto-generated description?
     name = example.full_description
     file = escape(example.metadata[:file_path])
@@ -129,9 +155,11 @@ class TrunkAnalyticsListener
 
     attempt_number = example.metadata[:attempt_number] || 0
     status = example.execution_result.status.to_s
+    # set the status to failure, but mark it as quarantined
+    is_quarantined = example.metadata[:quarantined_exception] ? true : false
     case example.execution_result.status
     when :passed
-      status = Status.new('success')
+      status = is_quarantined ? Status.new('failure') : Status.new('success')
     when :failed
       status = Status.new('failure')
     when :pending
@@ -140,12 +168,12 @@ class TrunkAnalyticsListener
     parent_name = example.example_group.metadata[:description]
     parent_name = parent_name.empty? ? 'rspec' : parent_name
     @testreport.add_test(id, name, classname, file, parent_name, line, status, attempt_number,
-                         started_at, finished_at, failure_message || '')
+                         started_at, finished_at, failure_message || '', is_quarantined)
   end
 end
 
 RSpec.configure do |c|
-  next if ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true'
+  next if trunk_disabled
 
   c.reporter.register_listener TrunkAnalyticsListener.new, :example_finished, :close
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec_trunk_flaky_tests
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.5
 platform: x86_64-darwin
 authors:
 - Trunk Technologies, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-03-07 00:00:00.000000000 Z
+date: 2025-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core