rspec_trunk_flaky_tests 0.7.2-aarch64-linux → 0.7.6-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 462d675759757f4979513d9a64585b881e6f94e23303ebc00da2466220341b13
-  data.tar.gz: 5b19da17f59d07b363fdc59c42905014a15f965a18deb64ef01f42db9411fdf0
+  metadata.gz: '036314043582f4d12501984763da8b39cd62a36d9056749eb46ce8eae443655a'
+  data.tar.gz: aae1bacdf05bdd611f77ae62345a2e041eadef4330f0ab1568606d43196fb15c
 SHA512:
-  metadata.gz: 401282baa06e83334940240b359ebc74a8964488103d5a324a00a4a161bb697060d636d5fe9ba0fbb9c5e1832da3b7c143e786173652a64c9c467fdafa648744
-  data.tar.gz: 767d119c261e9124f46b82a18fc2006dac8438e529b0795807e010e1d11235f880e5ee2abac25fbe67e6ce095615039ab2b614766e26a5f407a5ccc11816a225
+  metadata.gz: af47f814fa2a1e7267ba101d72364a0a1904a97fdeedef7fbf730ab71b51f8235bf5965a411165d23d723f1498d6e2d216ea5e8f681a6fc5b075df13d029eed9
+  data.tar.gz: 3d5f0250cbb6e884209dd578208eb0d79b685ba248a727e58ca68283db322da9c62b502c54d5013215967f702cd068ba9b3ed462c39a36b783bec2370fe8ab36

data/lib/trunk_spec_helper.rb

@@ -4,10 +4,49 @@ require 'rspec/core'
 require 'time'
 require 'context_ruby'
 
+# String is an override to the main String class that is used to colorize the output
+# it is used to make the output more readable
+class String
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def yellow
+    colorize(33)
+  end
+end
+
 def escape(str)
   str.dump[1..-2]
 end
 
+def description_generated?(example)
+  auto_generated_exp = /^\s?is expected to .*$/
+  full_description = example.full_description
+  parent_description = example.example_group.description
+  checked_description = full_description.sub(parent_description, '')
+  !auto_generated_exp.match(checked_description).nil? || full_description.empty?
+end
+
+def generate_id(example)
+  return "trunk:#{example.id}-#{example.location}" if description_generated?(example)
+end
+
+def trunk_disabled
+  ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' || ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil?
+end
+
+# we want to cache the test report so we can add to it as we go and reduce the number of API calls
+$test_report = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}")
+
 module RSpec
   module Core
     # Example is the class that represents a test case
@@ -17,16 +56,25 @@ module RSpec
       # RSpec uses the existance of an exception to determine if the test failed
       # We need to override this to allow us to capture the exception and then
       # decide if we want to fail the test or not
-      # trunk-ignore(rubocop/Naming/AccessorMethodName)
+      # trunk-ignore(rubocop/Naming/AccessorMethodName,rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
       def set_exception(exception)
-        # TODO: this is where we'll need to override the result once the logic is ready
-        # trunk-ignore(rubocop/Lint/LiteralAsCondition)
-        if true
-          set_exception_core(exception)
-        else
+        return set_exception_core(exception) if trunk_disabled
+
+        id = generate_id(self)
+        name = full_description
+        parent_name = example_group.metadata[:description]
+        parent_name = parent_name.empty? ? 'rspec' : parent_name
+        file = escape(metadata[:file_path])
+        classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
+        puts "Test failed, checking if it can be quarantined: `#{location}`".yellow
+        if $test_report.is_quarantined(id, name, parent_name, classname, file)
           # monitor the override in the metadata
           metadata[:quarantined_exception] = exception
+          puts "Test is quarantined, overriding exception: #{exception}".green
           nil
+        else
+          puts 'Test is not quarantined, continuing'.red
+          set_exception_core(exception)
         end
       end
 
@@ -45,7 +93,7 @@ module RSpec
   class Trunk
     def self.setup
       RSpec.configure do |config|
-        if ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true'
+        if trunk_disabled
           config.around(:each, &:run)
         else
           config.around(:each, &:run_with_trunk)
@@ -70,10 +118,6 @@ module RSpec
       else
         @example.metadata[:attempt_number] = 0
       end
-
-      # add the test to the report
-      # return the report
-      @testreport
     end
   end
 end
@@ -82,7 +126,7 @@ end
 # it generates and submits the final test reports
 class TrunkAnalyticsListener
   def initialize
-    @testreport = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}")
+    @testreport = $test_report
   end
 
   def example_finished(notification)
@@ -90,34 +134,18 @@ class TrunkAnalyticsListener
   end
 
   def close(_notification)
-    @testreport.publish
-  end
-
-  def description_generated?(example)
-    auto_generated_exp = /^\s?is expected to eq .*$/
-    full_description = example.full_description
-    parent_description = example.example_group.description
-    checked_description = full_description.sub(parent_description, '')
-    auto_generated_exp.match(checked_description) != nil
-  end
-
-  def generate_id(example)
-    if description_generated?(example)
-      # trunk-ignore(rubocop/Style/SoleNestedConditional)
-      return "trunk:#{example.id}-#{example.location}" if description_generated?(example)
+    res = @testreport.publish
+    if res
+      puts 'Flaky tests report upload complete'.green
+    else
+      puts 'Failed to publish flaky tests report'.red
     end
-    nil
   end
 
   # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength,rubocop/Metrics/CyclomaticComplexity)
   def add_test_case(example)
-    if example.exception
-      failure_message = example.exception.message
-      # failure details is far more robust than the message, but noiser
-      # if example.exception.backtrace
-      # failure_details = example.exception.backtrace.join('\n')
-      # end
-    end
+    failure_message = example.exception.to_s if example.exception
+    failure_message = example.metadata[:quarantined_exception].to_s if example.metadata[:quarantined_exception]
     # TODO: should we use concatenated string or alias when auto-generated description?
     name = example.full_description
     file = escape(example.metadata[:file_path])
@@ -129,9 +157,11 @@ class TrunkAnalyticsListener
 
     attempt_number = example.metadata[:attempt_number] || 0
     status = example.execution_result.status.to_s
+    # set the status to failure, but mark it as quarantined
+    is_quarantined = example.metadata[:quarantined_exception] ? true : false
     case example.execution_result.status
     when :passed
-      status = Status.new('success')
+      status = is_quarantined ? Status.new('failure') : Status.new('success')
     when :failed
       status = Status.new('failure')
     when :pending
@@ -140,12 +170,12 @@ class TrunkAnalyticsListener
     parent_name = example.example_group.metadata[:description]
     parent_name = parent_name.empty? ? 'rspec' : parent_name
     @testreport.add_test(id, name, classname, file, parent_name, line, status, attempt_number,
-                         started_at, finished_at, failure_message || '')
+                         started_at, finished_at, failure_message || '', is_quarantined)
   end
 end
 
 RSpec.configure do |c|
-  next if ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true'
+  next if trunk_disabled
 
   c.reporter.register_listener TrunkAnalyticsListener.new, :example_finished, :close
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec_trunk_flaky_tests
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.6
 platform: aarch64-linux
 authors:
 - Trunk Technologies, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-03-07 00:00:00.000000000 Z
+date: 2025-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core