rspec_trunk_flaky_tests 0.15.6-x86_64-linux-gnu → 0.16.0.pre.beta.1-x86_64-linux-gnu

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f311c33fb02453903e4ae41d750dfdfc847c30a819eb17e91effa65e3f9b1f89
-  data.tar.gz: 3e76f359ac2004a78b40331aeee873824aa6f3985d123472dcd6931041d016a9
+  metadata.gz: 7b52faa9ecd5bfa262776ef4d8f73de033dd4a891daf3ffc31153323b7a5c0cf
+  data.tar.gz: 4a9fef8855bfce5c976efbfcf55056d49166cf61da0fa7ef1d51123b2db6a44b
 SHA512:
-  metadata.gz: bcfe4e0f722687a9379e0365f067114d6208d06ee24e349425c9b10df57f026c2701274caeb70f544e22433775b3a8c6a01a013792c1e9f569496bd4e91ccd69
-  data.tar.gz: 026b982e02a44d14bf7e888da13d8404ef96fd76f811c882bca95cdb06ad11e0893c5583502910f5ac8b605aa5cd44c33892352b4957cf976f267a43c8fd4960
+  metadata.gz: 67dffdb5d3484cec95d2604776ed03a227e0a540f034e48b6183cdc0480fe5e3b6f917eb0d74a84b7763bee240280e31c40159801b18e67a0f006416d89a1cf1
+  data.tar.gz: ba96e766399fb7e6944f2f9deb9659e584409757de75e07da0dedd821e62767790883a2335c45dddca129cf0d34424455a732a268e08883279ecd03020102aae

data/lib/trunk_spec_helper.rb

@@ -37,407 +37,438 @@ require 'rspec/core/formatters/exception_presenter'
 require 'time'
 require 'rspec_trunk_flaky_tests'
 
-# trunk-ignore-all(rubocop/Style/GlobalVars)
+module RSpec
+  # Trunk holds everything this gem defines (the native extension defines the
+  # module and its classes: TestReport, Status, CIInfo, ...). Nothing is added to
+  # the global namespace or to core classes; test/namespace_spec.rb enforces that.
+  module Trunk
+    ANSI_ESCAPE_PATTERN = %r{(?:\e[@-Z\\-_]|\e\[[0-?]*[ -/]*[@-~])}
+
+    # Knapsack example detector instantiates all test cases in order to determine how to shard them
+    # These instantiations should not generate test bundles, so we
+    # disable the gem when running under knapsack_pro:rspec_test_example_detector
+    KNAPSACK_DETECTOR_COMMANDS = %w[
+      knapsack_pro:rspec_test_example_detector
+      knapsack_pro:queue:rspec:initialize
+    ].freeze
+
+    # Example metadata keys this gem sets, each mapped to the unprefixed name it
+    # used to be set under. The old names are deprecated: they are still written
+    # (see .write_metadata) so existing readers keep working, but will be removed
+    # in a future release.
+    DEPRECATED_METADATA_KEYS = {
+      trunk_quarantined_exception: :quarantined_exception,
+      trunk_attempt_number: :attempt_number,
+      trunk_is_description_generated: :is_description_generated
+    }.freeze
+
+    class << self
+      # The state for this RSpec process, or nil when Trunk is disabled.
+      attr_reader :current_run
+
+      def setup
+        return if current_run || disabled?
+
+        run = @current_run = Run.new
+        RSpec::Core::Example.prepend(ExampleExtension)
+        RSpec.configure do |config|
+          config.before(:example) do
+            skip('Quarantine lookup failed, skipping test run') if run.abort_remaining?
+          end
+          config.around(:each) { |example| Trunk.run_counting_attempts(example) }
+          config.reporter.register_listener AnalyticsListener.new(run), :example_finished, :close
+        end
+      end
 
-# String is an override to the main String class that is used to colorize the output
-# it is used to make the output more readable
-class String
-  def colorize(color_code)
-    "\e[#{color_code}m#{self}\e[0m"
-  end
+      def disabled?
+        knapsack_detector_mode? || ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' ||
+          ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil?
+      end
 
-  def red
-    colorize(31)
-  end
+      def knapsack_detector_mode?
+        KNAPSACK_DETECTOR_COMMANDS.any? { |command| command_line.include?(command) }
+      end
 
-  def green
-    colorize(32)
-  end
+      def quarantine_query_failure_exit?
+        ENV['TRUNK_QUARANTINE_QUERY_FAILURE_EXIT'] == 'true'
+      end
 
-  def yellow
-    colorize(33)
-  end
-end
+      def command_line
+        "#{$PROGRAM_NAME} #{ARGV.join(' ')}"
+      end
 
-ANSI_ESCAPE_PATTERN = %r{(?:\e[@-Z\\-_]|\e\[[0-?]*[ -/]*[@-~])}
+      def run_counting_attempts(example)
+        example.run
+        # monitor attempts in the metadata
+        attempt_number = example.metadata[:trunk_attempt_number]
+        write_metadata(example.metadata, :trunk_attempt_number, attempt_number ? attempt_number + 1 : 0)
+      end
 
-def escape(str)
-  str.dump[1..-2]
-end
+      # Sets a trunk_ metadata key, and its deprecated unprefixed name too.
+      def write_metadata(metadata, key, value)
+        metadata[key] = value
+        deprecated_key = DEPRECATED_METADATA_KEYS[key]
+        metadata[deprecated_key] = value if deprecated_key
+      end
 
-def strip_ansi_codes(text)
-  text.to_s.gsub(ANSI_ESCAPE_PATTERN, '')
-end
+      def escape(str)
+        str.dump[1..-2]
+      end
 
-# Knapsack example detector instantiates all test cases in order to determine how to shard them
-# These instantiations should not generate test bundles, so we
-# disable the gem when running under knapsack_pro:rspec_test_example_detector
-def knapsack_detector_mode?
-  knapsack_detector_command?
-end
+      def strip_ansi_codes(text)
+        text.to_s.gsub(ANSI_ESCAPE_PATTERN, '')
+      end
 
-def knapsack_detector_command?
-  command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip
-  command_line.include?('knapsack_pro:rspec_test_example_detector') ||
-    command_line.include?('knapsack_pro:queue:rspec:initialize')
-end
+      # The escaped file path and the dotted classname derived from it.
+      def file_and_classname(example)
+        file = escape(example.metadata[:file_path])
+        [file, file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')]
+      end
 
-def trunk_disabled
-  knapsack_detector_mode? || ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' ||
-    ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil?
-end
+      def parent_name(example)
+        name = example.example_group.metadata[:description]
+        name.empty? ? 'rspec' : name
+      end
+    end
 
-def quarantine_query_failure_exit?
-  ENV['TRUNK_QUARANTINE_QUERY_FAILURE_EXIT'] == 'true'
-end
+    # ANSI colors for the gem's console output, without patching String.
+    module Colors
+      module_function
 
-# we want to cache the test report in memory so we can add to it as we go and reduce the number of API calls
-$test_report = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}", nil)
-$failure_encountered_and_quarantining_disabled = false
-$failure_encountered_and_quarantine_lookup_failed = false
+      def colorize(text, color_code)
+        "\e[#{color_code}m#{text}\e[0m"
+      end
 
-module RSpec
-  module Core
-    # Example is the class that represents a test case
-    class Example
-      # keep the original method around so we can call it
-      alias set_exception_core set_exception
-      alias assign_generated_description_core assign_generated_description
-      # RSpec uses the existance of an exception to determine if the test failed
-      # We need to override this to allow us to capture the exception and then
-      # decide if we want to fail the test or not
-      # trunk-ignore(rubocop/Naming/AccessorMethodName)
-      def set_exception(exception)
-        # Pending stays green, except a fixed pending example is a real failure.
-        return set_exception_core(exception) if metadata[:pending] && !pending_example_fixed?(exception)
-        return set_exception_core(exception) if trunk_disabled
-        return set_exception_core(exception) if metadata[:retry_attempts]&.positive?
+      def red(text)
+        colorize(text, 31)
+      end
 
-        handle_quarantine_check_safely(exception)
+      def green(text)
+        colorize(text, 32)
       end
 
-      # If the quarantine machinery itself blows up, the failure must stand.
-      def handle_quarantine_check_safely(exception)
-        handle_quarantine_check(exception)
-      rescue StandardError => e
-        puts "Quarantine check errored (#{e.class}: #{e.message}), treating test as not quarantined".yellow
-        set_exception_core(exception)
+      def yellow(text)
+        colorize(text, 33)
       end
+    end
 
-      def pending_example_fixed?(exception)
-        defined?(RSpec::Core::Pending::PendingExampleFixedError) &&
-          exception.is_a?(RSpec::Core::Pending::PendingExampleFixedError)
+    # Run is the state shared by every example in this RSpec process: the test
+    # report (cached in memory so we can add to it as we go and reduce the number
+    # of API calls), and whether quarantining turned out to be unavailable, so
+    # that's only announced once.
+    class Run
+      attr_reader :report
+
+      def initialize(report = TestReport.new('rspec', Trunk.command_line, nil))
+        @report = report
+        @quarantining_disabled = false
+        @lookup_failed = false
+      end
+
+      def quarantining_disabled?
+        @quarantining_disabled
+      end
+
+      def lookup_failed?
+        @lookup_failed
+      end
+
+      def abort_remaining?
+        lookup_failed? && Trunk.quarantine_query_failure_exit?
+      end
+
+      # Whether a failure of `example` with `exception` is quarantined. If the
+      # quarantine machinery itself blows up, the failure must stand.
+      def quarantine?(example, exception)
+        check_quarantine(example, exception)
+      rescue StandardError => e
+        puts Colors.yellow("Quarantine check errored (#{e.class}: #{e.message}), treating test as not quarantined")
+        false
       end
 
+      private
+
       # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength,rubocop/Metrics/CyclomaticComplexity)
-      def handle_quarantine_check(exception)
-        id = generate_trunk_id
-        name = full_description
-        parent_name = example_group.metadata[:description]
-        parent_name = parent_name.empty? ? 'rspec' : parent_name
-        file = escape(metadata[:file_path])
-        classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
-        unless $failure_encountered_and_quarantining_disabled
-          puts "Test failed, checking if it can be quarantined: `#{location}`".yellow
+      def check_quarantine(example, exception)
+        file, classname = Trunk.file_and_classname(example)
+        unless quarantining_disabled?
+          puts Colors.yellow("Test failed, checking if it can be quarantined: `#{example.location}`")
         end
-        is_quarantined_result = $test_report.is_quarantined(id, name, parent_name, classname, file)
+        result = report.is_quarantined(example.trunk_id, example.full_description, Trunk.parent_name(example),
+                                       classname, file)
 
-        if is_quarantined_result.quarantine_lookup_failed
-          unless $failure_encountered_and_quarantine_lookup_failed
-            puts 'Failed to check quarantining status, no failures will be quarantined'.yellow
-            $failure_encountered_and_quarantine_lookup_failed = true
+        if result.quarantine_lookup_failed
+          unless lookup_failed?
+            puts Colors.yellow('Failed to check quarantining status, no failures will be quarantined')
+            @lookup_failed = true
           end
-          if quarantine_query_failure_exit?
-            puts 'Quarantine lookup failed, exiting early'.red
+          if Trunk.quarantine_query_failure_exit?
+            puts Colors.red('Quarantine lookup failed, exiting early')
             RSpec.world.wants_to_quit = true
           end
-          set_exception_core(exception)
-        elsif is_quarantined_result.quarantining_disabled_for_repo
-          unless $failure_encountered_and_quarantining_disabled
-            puts 'Quarantining is disabled for this repo, no failures will be quarantined'.yellow
-            $failure_encountered_and_quarantining_disabled = true
+          false
+        elsif result.quarantining_disabled_for_repo
+          unless quarantining_disabled?
+            puts Colors.yellow('Quarantining is disabled for this repo, no failures will be quarantined')
+            @quarantining_disabled = true
           end
-          set_exception_core(exception)
-        elsif is_quarantined_result.test_is_quarantined
+          false
+        elsif result.test_is_quarantined
           # monitor the override in the metadata
-          metadata[:quarantined_exception] = exception
-          puts "Test is quarantined, overriding exception: #{exception}".green
-          nil
+          Trunk.write_metadata(example.metadata, :trunk_quarantined_exception, exception)
+          puts Colors.green("Test is quarantined, overriding exception: #{exception}")
+          true
         else
-          puts 'Test is not quarantined, continuing'.red
-          set_exception_core(exception)
+          puts Colors.red('Test is not quarantined, continuing')
+          false
         end
       end
+    end
 
-      def assign_generated_description
-        metadata[:is_description_generated] = description_generated?
-        assign_generated_description_core
-      end
-
-      def description_generated?
-        return metadata[:is_description_generated] unless metadata[:is_description_generated].nil?
+    # Prepended to RSpec::Core::Example. RSpec uses the existence of an exception
+    # to determine if the test failed, so #set_exception is where we capture the
+    # exception and decide whether to fail the test or not.
+    module ExampleExtension
+      # trunk-ignore(rubocop/Naming/AccessorMethodName)
+      def set_exception(exception)
+        return super unless quarantine_candidate?(exception)
+        return super unless Trunk.current_run&.quarantine?(self, exception)
 
-        description == location_description
+        nil
       end
 
-      def generate_trunk_id
-        return "trunk:#{id}-#{location}" if description_generated?
+      def trunk_id
+        "trunk:#{id}-#{location}" if trunk_description_generated?
       end
 
-      # Procsy is a class that is used to wrap execution of the Example class
-      class Procsy
-        def run_with_trunk
-          RSpec::Trunk.new(self).run
-        end
-      end
-    end
-  end
-end
+      private
 
-module RSpec
-  # Trunk is a class that is used to monitor the execution of the Example class
-  class Trunk
-    def self.setup
-      RSpec.configure do |config|
-        if trunk_disabled
-          config.around(:each, &:run)
-        else
-          config.around(:each, &:run_with_trunk)
-        end
+      def assign_generated_description
+        Trunk.write_metadata(metadata, :trunk_is_description_generated, trunk_description_generated?)
+        super
       end
-    end
-
-    def initialize(example)
-      @example = example
-    end
 
-    def current_example
-      @current_example ||= RSpec.current_example
-    end
+      def trunk_description_generated?
+        cached = metadata[:trunk_is_description_generated]
+        return cached unless cached.nil?
 
-    def run
-      # run the test
-      @example.run
-      # monitor attempts in the metadata
-      if @example.metadata[:attempt_number]
-        @example.metadata[:attempt_number] += 1
-      else
-        @example.metadata[:attempt_number] = 0
+        description == location_description
       end
-    end
-  end
-end
 
-# PlainColorizer is a no-op colorizer passed to RSpec's ExceptionPresenter so the
-# formatted output is plain text suitable for storage and the web UI.
-module PlainColorizer
-  module_function
+      def quarantine_candidate?(exception)
+        # Pending stays green, except a fixed pending example is a real failure.
+        return false if metadata[:pending] && !pending_example_fixed?(exception)
 
-  def wrap(text, _code_or_symbol)
-    text
-  end
-end
+        !metadata[:retry_attempts]&.positive?
+      end
 
-# Defer to RSpec's own ExceptionPresenter so the failure_message field matches
-# what users see in their RSpec console output (Failure/Error: <source line>,
-# the exception class and message, and any "Caused by:" chain).
-def format_exception_message(exception, example)
-  return '' unless exception
+      def pending_example_fixed?(exception)
+        defined?(RSpec::Core::Pending::PendingExampleFixedError) &&
+          exception.is_a?(RSpec::Core::Pending::PendingExampleFixedError)
+      end
+    end
 
-  presenter = RSpec::Core::Formatters::ExceptionPresenter.new(exception, example)
-  strip_ansi_codes(presenter.fully_formatted(nil, PlainColorizer))
-rescue StandardError
-  legacy_format_exception_message(exception)
-end
+    # Formats failures for the report as plain text suitable for storage and the web UI.
+    module FailureFormatter
+      # A no-op colorizer passed to RSpec's ExceptionPresenter.
+      module PlainColorizer
+        module_function
 
-# trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
-def format_exception_backtrace(exception, example)
-  return '' unless exception
+        def wrap(text, _code_or_symbol)
+          text
+        end
+      end
 
-  lines = exception_backtrace_lines(exception, example)
+      module_function
 
-  cause = exception.cause
-  depth = 0
-  while cause && depth < 10
-    lines << ''
-    lines << "Caused by: #{cause.class}: #{cause.message}"
-    lines.concat(exception_backtrace_lines(cause, example))
-    cause = cause.cause
-    depth += 1
-  end
+      # Defer to RSpec's own ExceptionPresenter so the failure_message field matches
+      # what users see in their RSpec console output (Failure/Error: <source line>,
+      # the exception class and message, and any "Caused by:" chain).
+      def message(exception, example)
+        return '' unless exception
 
-  result = lines.join("\n")
-  # The exception presenter may choke on MultipleExceptionError, such as errors in before
-  # and after hooks, so we fall back to the legacy formatter
-  return legacy_format_exception_backtrace(exception) if result.strip.empty?
+        presenter = RSpec::Core::Formatters::ExceptionPresenter.new(exception, example)
+        Trunk.strip_ansi_codes(presenter.fully_formatted(nil, PlainColorizer))
+      rescue StandardError
+        legacy_message(exception)
+      end
 
-  strip_ansi_codes(result)
-rescue StandardError
-  legacy_format_exception_backtrace(exception)
-end
+      # trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
+      def backtrace(exception, example)
+        return '' unless exception
 
-def exception_backtrace_lines(exception, example)
-  presenter = RSpec::Core::Formatters::ExceptionPresenter.new(exception, example)
-  Array(presenter.formatted_backtrace)
-rescue StandardError
-  Array(exception.backtrace)
-end
+        lines = backtrace_lines(exception, example)
 
-def legacy_format_exception_message(exception)
-  case exception
-  when RSpec::Core::MultipleExceptionError
-    messages = exception.all_exceptions.map { |e| "#{e.class}: #{e.message}" }
-    strip_ansi_codes("#{exception.class}: #{messages.join(' | ')}")
-  else
-    strip_ansi_codes(exception.to_s)
-  end
-end
+        cause = exception.cause
+        depth = 0
+        while cause && depth < 10
+          lines << ''
+          lines << "Caused by: #{cause.class}: #{cause.message}"
+          lines.concat(backtrace_lines(cause, example))
+          cause = cause.cause
+          depth += 1
+        end
 
-# trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
-def legacy_format_exception_backtrace(exception)
-  case exception
-  when RSpec::Core::MultipleExceptionError
-    strip_ansi_codes(exception.all_exceptions.map do |e|
-      if e.backtrace && !e.backtrace.empty?
-        "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
-      else
-        "#{e.class}: #{e.message}"
-      end
-    end.join("\n\n"))
-  else
-    strip_ansi_codes(exception.backtrace&.join("\n") || '')
-  end
-end
+        result = lines.join("\n")
+        # The exception presenter may choke on MultipleExceptionError, such as errors in before
+        # and after hooks, so we fall back to the legacy formatter
+        return legacy_backtrace(exception) if result.strip.empty?
 
-# TrunkAnalyticsListener is a class that is used to listen to the execution of the Example class
-# it generates and submits the final test reports
-class TrunkAnalyticsListener
-  MAX_TEXT_FIELD_SIZE = 8_000
+        Trunk.strip_ansi_codes(result)
+      rescue StandardError
+        legacy_backtrace(exception)
+      end
 
-  def initialize
-    @testreport = $test_report
-  end
+      def backtrace_lines(exception, example)
+        presenter = RSpec::Core::Formatters::ExceptionPresenter.new(exception, example)
+        Array(presenter.formatted_backtrace)
+      rescue StandardError
+        Array(exception.backtrace)
+      end
 
-  def example_finished(notification)
-    add_test_case(notification.example)
-  end
+      def legacy_message(exception)
+        case exception
+        when RSpec::Core::MultipleExceptionError
+          messages = exception.all_exceptions.map { |e| "#{e.class}: #{e.message}" }
+          Trunk.strip_ansi_codes("#{exception.class}: #{messages.join(' | ')}")
+        else
+          Trunk.strip_ansi_codes(exception.to_s)
+        end
+      end
 
-  # trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
-  def close(_notification)
-    if $failure_encountered_and_quarantining_disabled
-      puts 'Note: Quarantining is disabled for this repo. Test failures were not quarantined.'.yellow
-    end
-    if $failure_encountered_and_quarantine_lookup_failed
-      puts 'Note: Failed to check quarantining status. Test failures were not quarantined.'.yellow
+      # trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
+      def legacy_backtrace(exception)
+        case exception
+        when RSpec::Core::MultipleExceptionError
+          Trunk.strip_ansi_codes(exception.all_exceptions.map do |e|
+            if e.backtrace && !e.backtrace.empty?
+              "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+            else
+              "#{e.class}: #{e.message}"
+            end
+          end.join("\n\n"))
+        else
+          Trunk.strip_ansi_codes(exception.backtrace&.join("\n") || '')
+        end
+      end
     end
 
-    if ENV['TRUNK_LOCAL_UPLOAD_DIR']
-      saved = @testreport.try_save(ENV['TRUNK_LOCAL_UPLOAD_DIR'])
-      if saved
-        puts 'Local Flaky tests report generated'.green
-      else
-        puts 'Failed to generate local flaky tests report'.red
+    # AnalyticsListener is an RSpec reporter listener that records each finished
+    # example in the run's test report and submits the report at the end.
+    class AnalyticsListener
+      MAX_TEXT_FIELD_SIZE = 8_000
+
+      def initialize(run)
+        @run = run
       end
-    else
-      published = @testreport.publish
-      if published
-        puts 'Flaky tests report upload complete'.green
-      else
-        puts 'Failed to publish flaky tests report'.red
+
+      def example_finished(notification)
+        add_test_case(notification.example)
       end
-    end
-  end
 
-  # trunk-ignore(rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength)
-  def add_test_case(example)
-    status, exception = status_and_exception(example)
-    failure_message = ''
-    backtrace = ''
-    if exception
-      failure_message = format_exception_message(exception, example).strip
-      backtrace = format_exception_backtrace(exception, example).strip
-    end
-    failure_message = failure_message[0...MAX_TEXT_FIELD_SIZE] if failure_message.length > MAX_TEXT_FIELD_SIZE
-    backtrace = backtrace[0...MAX_TEXT_FIELD_SIZE] if backtrace.length > MAX_TEXT_FIELD_SIZE
-    # TODO: should we use concatenated string or alias when auto-generated description?
-    name = example.full_description
-    file = escape(example.metadata[:file_path])
-    classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
-    line = example.metadata[:line_number]
-    started_at = example.execution_result.started_at.to_i
-    finished_at = example.execution_result.finished_at.to_i
-    id = example.generate_trunk_id
-
-    attempt_number = example.metadata[:retry_attempts] || example.metadata[:attempt_number] || 0
-    # set the status to failure, but mark it as quarantined
-    is_quarantined = example.metadata[:quarantined_exception] ? true : false
-    parent_name = example.example_group.metadata[:description]
-    parent_name = parent_name.empty? ? 'rspec' : parent_name
-    @testreport.add_test(id, name, classname, file, parent_name, line, status, attempt_number,
-                         started_at, finished_at, failure_message || '', backtrace || '', is_quarantined)
-  end
+      # trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
+      def close(_notification)
+        if @run.quarantining_disabled?
+          puts Colors.yellow('Note: Quarantining is disabled for this repo. Test failures were not quarantined.')
+        end
+        if @run.lookup_failed?
+          puts Colors.yellow('Note: Failed to check quarantining status. Test failures were not quarantined.')
+        end
 
-  # Determine the Trunk status to report for an example, plus the exception (if
-  # any) whose message/backtrace should be recorded for it.
-  #
-  # `pending` examples need care because RSpec expresses their outcome as the
-  # inverse of the body's pass/fail. A pending example's contract is "this is
-  # expected to fail", so:
-  #   - body still fails  -> expectation met     -> RSpec status :pending, build green
-  #   - body now passes   -> expectation violated -> RSpec status :failed
-  #                          (PendingExampleFixedError), build red
-  # We report the outcome RSpec actually decided, which also matches the build
-  # result:
-  #   - skip / xit (never ran)     -> skipped
-  #   - pending, body still failing -> success (expectation met)
-  #   - pending, body now passing   -> failure (PendingExampleFixedError)
-  # RSpec's own build pass/fail behavior is left untouched (see #set_exception).
-  #
-  # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/MethodLength)
-  def status_and_exception(example)
-    result = example.execution_result
-    quarantined_exception = example.metadata[:quarantined_exception]
-
-    # A genuinely skipped example (`skip`/`xit`, or `skip` called from a hook or
-    # body) never runs, so it has no real pass/fail outcome. RSpec still reports
-    # it with status :pending, so detect skips explicitly -- and up front -- via
-    # metadata[:skip] before interpreting any pending pass/fail semantics below.
-    return [Status.new('skipped'), nil] if example.metadata[:skip]
-
-    case result.status
-    when :passed
-      # A quarantined failure is recorded as :passed by RSpec (see #set_exception),
-      # so report it as a failure but carry the original quarantined exception.
-      quarantined_exception ? [Status.new('failure'), quarantined_exception] : [Status.new('success'), nil]
-    when :failed
-      # Includes a pending example whose body unexpectedly passed: RSpec reports it
-      # as :failed with a PendingExampleFixedError (on example.exception) and breaks
-      # the build (the pending expectation was violated), so it is a failure here too.
-      [Status.new('failure'), example.exception || quarantined_exception]
-    when :pending
-      # A quarantined fixed-pending is left :pending but pending_fixed? -- still a
-      # failure. Otherwise the pending "should fail" expectation was met -> success.
-      if result.pending_fixed?
-        [Status.new('failure'), quarantined_exception || example.exception]
-      else
-        [Status.new('success'), nil]
-      end
-    else
-      [Status.new(result.status.to_s), example.exception || quarantined_exception]
-    end
-  end
-end
+        if ENV['TRUNK_LOCAL_UPLOAD_DIR']
+          saved = @run.report.try_save(ENV['TRUNK_LOCAL_UPLOAD_DIR'])
+          if saved
+            puts Colors.green('Local Flaky tests report generated')
+          else
+            puts Colors.red('Failed to generate local flaky tests report')
+          end
+        else
+          published = @run.report.publish
+          if published
+            puts Colors.green('Flaky tests report upload complete')
+          else
+            puts Colors.red('Failed to publish flaky tests report')
+          end
+        end
+      end
 
-RSpec.configure do |c|
-  next if trunk_disabled
+      # trunk-ignore(rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength)
+      def add_test_case(example)
+        status, exception = status_and_exception(example)
+        failure_message = ''
+        backtrace = ''
+        if exception
+          failure_message = FailureFormatter.message(exception, example).strip
+          backtrace = FailureFormatter.backtrace(exception, example).strip
+        end
+        failure_message = failure_message[0...MAX_TEXT_FIELD_SIZE] if failure_message.length > MAX_TEXT_FIELD_SIZE
+        backtrace = backtrace[0...MAX_TEXT_FIELD_SIZE] if backtrace.length > MAX_TEXT_FIELD_SIZE
+        # TODO: should we use concatenated string or alias when auto-generated description?
+        name = example.full_description
+        file, classname = Trunk.file_and_classname(example)
+        line = example.metadata[:line_number]
+        started_at = example.execution_result.started_at.to_i
+        finished_at = example.execution_result.finished_at.to_i
+
+        attempt_number = example.metadata[:retry_attempts] || example.metadata[:trunk_attempt_number] || 0
+        # set the status to failure, but mark it as quarantined
+        is_quarantined = example.metadata[:trunk_quarantined_exception] ? true : false
+        @run.report.add_test(example.trunk_id, name, classname, file, Trunk.parent_name(example), line, status,
+                             attempt_number, started_at, finished_at, failure_message || '', backtrace || '',
+                             is_quarantined)
+      end
 
-  c.before(:example) do
-    if $failure_encountered_and_quarantine_lookup_failed && quarantine_query_failure_exit?
-      skip('Quarantine lookup failed, skipping test run')
+      # Determine the Trunk status to report for an example, plus the exception (if
+      # any) whose message/backtrace should be recorded for it.
+      #
+      # `pending` examples need care because RSpec expresses their outcome as the
+      # inverse of the body's pass/fail. A pending example's contract is "this is
+      # expected to fail", so:
+      #   - body still fails  -> expectation met     -> RSpec status :pending, build green
+      #   - body now passes   -> expectation violated -> RSpec status :failed
+      #                          (PendingExampleFixedError), build red
+      # We report the outcome RSpec actually decided, which also matches the build
+      # result:
+      #   - skip / xit (never ran)     -> skipped
+      #   - pending, body still failing -> success (expectation met)
+      #   - pending, body now passing   -> failure (PendingExampleFixedError)
+      # RSpec's own build pass/fail behavior is left untouched (see #set_exception).
+      #
+      # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/MethodLength)
+      def status_and_exception(example)
+        result = example.execution_result
+        quarantined_exception = example.metadata[:trunk_quarantined_exception]
+
+        # A genuinely skipped example (`skip`/`xit`, or `skip` called from a hook or
+        # body) never runs, so it has no real pass/fail outcome. RSpec still reports
+        # it with status :pending, so detect skips explicitly -- and up front -- via
+        # metadata[:skip] before interpreting any pending pass/fail semantics below.
+        return [Status.new('skipped'), nil] if example.metadata[:skip]
+
+        case result.status
+        when :passed
+          # A quarantined failure is recorded as :passed by RSpec (see #set_exception),
+          # so report it as a failure but carry the original quarantined exception.
+          quarantined_exception ? [Status.new('failure'), quarantined_exception] : [Status.new('success'), nil]
+        when :failed
+          # Includes a pending example whose body unexpectedly passed: RSpec reports it
+          # as :failed with a PendingExampleFixedError (on example.exception) and breaks
+          # the build (the pending expectation was violated), so it is a failure here too.
+          [Status.new('failure'), example.exception || quarantined_exception]
+        when :pending
+          # A quarantined fixed-pending is left :pending but pending_fixed? -- still a
+          # failure. Otherwise the pending "should fail" expectation was met -> success.
+          if result.pending_fixed?
+            [Status.new('failure'), quarantined_exception || example.exception]
+          else
+            [Status.new('success'), nil]
+          end
+        else
+          [Status.new(result.status.to_s), example.exception || quarantined_exception]
+        end
+      end
     end
   end
-
-  c.reporter.register_listener TrunkAnalyticsListener.new, :example_finished, :close
 end
 
 RSpec::Trunk.setup

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec_trunk_flaky_tests
 version: !ruby/object:Gem::Version
-  version: 0.15.6
+  version: 0.16.0.pre.beta.1
 platform: x86_64-linux-gnu
 authors:
 - Trunk Technologies, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2026-09-25 00:00:00.000000000 Z
+date: 2026-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core