rspec_trunk_flaky_tests 0.14.0.pre.beta.1-x86_64-linux-gnu

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 77070f7db8c691af81db14e4755d2b0a885e946f7baefa413c9178692e987b9d
+  data.tar.gz: a5b4b23deeef4d7aca34a7e2b9df28bb0a35ad215c938afd528b8e26ceed2e4c
+SHA512:
+  metadata.gz: b12a8825a7921ef88b17e5a8f3420e5276efc8d353a8e0a990ecb10dab72356551031543eb1c2a5ded9081e21ea4dee60bbba1ddcf9f531e2097316284bc67f2
+  data.tar.gz: 45cfa95cdab539509834da239b126b31ece68c331858af45b4a2e05d1ae6f79cb72500aec4248349eef87144497201a3bfa8fdc8310499db60aa0dc20079287c

data/lib/rspec_trunk_flaky_tests.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+begin
+  ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+  require_relative "rspec_trunk_flaky_tests/#{ruby_version}/rspec_trunk_flaky_tests"
+rescue LoadError
+  begin
+    require_relative 'rspec_trunk_flaky_tests/rspec_trunk_flaky_tests'
+  rescue LoadError
+    raise LoadError, <<~MSG
+      Could not load the native extension for rspec_trunk_flaky_tests.
+
+      You are using the pure ruby variant of this gem, which is a placeholder
+      that exists so that Gemfile.lock files with `PLATFORMS ruby` can resolve
+      this dependency. It does not include a precompiled native extension.
+
+      Precompiled native gems are available for:
+        x86_64-linux-gnu, x86_64-linux-musl, aarch64-linux-gnu,
+        aarch64-linux-musl, arm64-darwin, x86_64-darwin
+
+      If you are on a supported platform and seeing this error, make sure
+      bundler is selecting the native variant for your platform:
+
+          bundle lock --add-platform #{RUBY_PLATFORM}
+          bundle install
+
+      If you are on an unsupported platform, this gem cannot be used in your
+      environment. Please open an issue at:
+        https://github.com/trunk-io/analytics-cli/issues
+
+      Current platform: #{RUBY_PLATFORM}
+      Current Ruby version: #{RUBY_VERSION}
+    MSG
+  end
+end

data/lib/trunk_spec_helper.rb

@@ -0,0 +1,426 @@
+# frozen_string_literal: true
+
+# Trunk RSpec Helper
+#
+# This helper integrates Trunk Flaky Tests with RSpec to automatically
+# quarantine flaky tests and upload test results.
+#
+# Required environment variables:
+#   TRUNK_ORG_URL_SLUG - Your organization's URL slug
+#   TRUNK_API_TOKEN - Your API token for authentication
+#
+# Optional environment variables for repository metadata:
+#   TRUNK_REPO_ROOT - Path to repository root
+#   TRUNK_REPO_URL - Repository URL (e.g., https://github.com/org/repo.git)
+#   TRUNK_REPO_HEAD_SHA - HEAD commit SHA
+#   TRUNK_REPO_HEAD_BRANCH - HEAD branch name
+#   TRUNK_REPO_HEAD_COMMIT_EPOCH - HEAD commit timestamp (seconds since epoch)
+#   TRUNK_REPO_HEAD_AUTHOR_NAME - HEAD commit author name
+#   TRUNK_PR_NUMBER - PR number, if uploading from a PR (normally inferred from CI environment variables)
+#
+# Optional environment variables for configuration:
+#   TRUNK_CODEOWNERS_PATH - Path to CODEOWNERS file
+#   TRUNK_TEST_COLLECTION_ID - Optional 8 character alphanumeric ID for a test collection
+#   TRUNK_VARIANT - Variant name for test results (e.g., 'linux', 'pr-123')
+#   TRUNK_DISABLE_QUARANTINING - Set to 'true' to disable quarantining
+#   TRUNK_ALLOW_EMPTY_TEST_RESULTS - Set to 'true' to allow empty results
+#   TRUNK_DRY_RUN - Set to 'true' to save bundle locally instead of uploading
+#   TRUNK_USE_UNCLONED_REPO - Set to 'true' for uncloned repo mode
+#   TRUNK_LOCAL_UPLOAD_DIR - Directory to save test results locally (disables upload)
+#   TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds)
+#   TRUNK_QUARANTINE_QUERY_FAILURE_EXIT - Set to 'true' to abort the RSpec run when quarantine
+#     lookup fails (remaining examples are skipped)
+#   DISABLE_RSPEC_TRUNK_FLAKY_TESTS - Set to 'true' to completely disable Trunk
+#
+require 'rspec/core'
+require 'rspec/core/formatters/exception_presenter'
+require 'time'
+require 'rspec_trunk_flaky_tests'
+
+# trunk-ignore-all(rubocop/Style/GlobalVars)
+
+# String is an override to the main String class that is used to colorize the output
+# it is used to make the output more readable
+class String
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def yellow
+    colorize(33)
+  end
+end
+
+ANSI_ESCAPE_PATTERN = %r{(?:\e[@-Z\\-_]|\e\[[0-?]*[ -/]*[@-~])}
+
+def escape(str)
+  str.dump[1..-2]
+end
+
+def strip_ansi_codes(text)
+  text.to_s.gsub(ANSI_ESCAPE_PATTERN, '')
+end
+
+# Knapsack example detector instantiates all test cases in order to determine how to shard them
+# These instantiations should not generate test bundles, so we
+# disable the gem when running under knapsack_pro:rspec_test_example_detector
+def knapsack_detector_mode?
+  knapsack_detector_command?
+end
+
+def knapsack_detector_command?
+  command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip
+  command_line.include?('knapsack_pro:rspec_test_example_detector') ||
+    command_line.include?('knapsack_pro:queue:rspec:initialize')
+end
+
+def trunk_disabled
+  knapsack_detector_mode? || ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' ||
+    ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil?
+end
+
+def quarantine_query_failure_exit?
+  ENV['TRUNK_QUARANTINE_QUERY_FAILURE_EXIT'] == 'true'
+end
+
+# we want to cache the test report in memory so we can add to it as we go and reduce the number of API calls
+$test_report = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}", nil)
+$failure_encountered_and_quarantining_disabled = false
+$failure_encountered_and_quarantine_lookup_failed = false
+
+module RSpec
+  module Core
+    # Example is the class that represents a test case
+    class Example
+      # keep the original method around so we can call it
+      alias set_exception_core set_exception
+      alias assign_generated_description_core assign_generated_description
+      # RSpec uses the existance of an exception to determine if the test failed
+      # We need to override this to allow us to capture the exception and then
+      # decide if we want to fail the test or not
+      # trunk-ignore(rubocop/Naming/AccessorMethodName)
+      def set_exception(exception)
+        return set_exception_core(exception) if metadata[:pending]
+        return set_exception_core(exception) if trunk_disabled
+        return set_exception_core(exception) if metadata[:retry_attempts]&.positive?
+
+        handle_quarantine_check(exception)
+      end
+
+      # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength,rubocop/Metrics/CyclomaticComplexity)
+      def handle_quarantine_check(exception)
+        id = generate_trunk_id
+        name = full_description
+        parent_name = example_group.metadata[:description]
+        parent_name = parent_name.empty? ? 'rspec' : parent_name
+        file = escape(metadata[:file_path])
+        classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
+        unless $failure_encountered_and_quarantining_disabled
+          puts "Test failed, checking if it can be quarantined: `#{location}`".yellow
+        end
+        is_quarantined_result = $test_report.is_quarantined(id, name, parent_name, classname, file)
+
+        if is_quarantined_result.quarantine_lookup_failed
+          unless $failure_encountered_and_quarantine_lookup_failed
+            puts 'Failed to check quarantining status, no failures will be quarantined'.yellow
+            $failure_encountered_and_quarantine_lookup_failed = true
+          end
+          if quarantine_query_failure_exit?
+            puts 'Quarantine lookup failed, exiting early'.red
+            RSpec.world.wants_to_quit = true
+          end
+          set_exception_core(exception)
+        elsif is_quarantined_result.quarantining_disabled_for_repo
+          unless $failure_encountered_and_quarantining_disabled
+            puts 'Quarantining is disabled for this repo, no failures will be quarantined'.yellow
+            $failure_encountered_and_quarantining_disabled = true
+          end
+          set_exception_core(exception)
+        elsif is_quarantined_result.test_is_quarantined
+          # monitor the override in the metadata
+          metadata[:quarantined_exception] = exception
+          puts "Test is quarantined, overriding exception: #{exception}".green
+          nil
+        else
+          puts 'Test is not quarantined, continuing'.red
+          set_exception_core(exception)
+        end
+      end
+
+      def assign_generated_description
+        metadata[:is_description_generated] = description_generated?
+        assign_generated_description_core
+      end
+
+      def description_generated?
+        return metadata[:is_description_generated] unless metadata[:is_description_generated].nil?
+
+        description == location_description
+      end
+
+      def generate_trunk_id
+        return "trunk:#{id}-#{location}" if description_generated?
+      end
+
+      # Procsy is a class that is used to wrap execution of the Example class
+      class Procsy
+        def run_with_trunk
+          RSpec::Trunk.new(self).run
+        end
+      end
+    end
+  end
+end
+
+module RSpec
+  # Trunk is a class that is used to monitor the execution of the Example class
+  class Trunk
+    def self.setup
+      RSpec.configure do |config|
+        if trunk_disabled
+          config.around(:each, &:run)
+        else
+          config.around(:each, &:run_with_trunk)
+        end
+      end
+    end
+
+    def initialize(example)
+      @example = example
+    end
+
+    def current_example
+      @current_example ||= RSpec.current_example
+    end
+
+    def run
+      # run the test
+      @example.run
+      # monitor attempts in the metadata
+      if @example.metadata[:attempt_number]
+        @example.metadata[:attempt_number] += 1
+      else
+        @example.metadata[:attempt_number] = 0
+      end
+    end
+  end
+end
+
+# PlainColorizer is a no-op colorizer passed to RSpec's ExceptionPresenter so the
+# formatted output is plain text suitable for storage and the web UI.
+module PlainColorizer
+  module_function
+
+  def wrap(text, _code_or_symbol)
+    text
+  end
+end
+
+# Defer to RSpec's own ExceptionPresenter so the failure_message field matches
+# what users see in their RSpec console output (Failure/Error: <source line>,
+# the exception class and message, and any "Caused by:" chain).
+def format_exception_message(exception, example)
+  return '' unless exception
+
+  presenter = RSpec::Core::Formatters::ExceptionPresenter.new(exception, example)
+  strip_ansi_codes(presenter.fully_formatted(nil, PlainColorizer))
+rescue StandardError
+  legacy_format_exception_message(exception)
+end
+
+# trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
+def format_exception_backtrace(exception, example)
+  return '' unless exception
+
+  lines = exception_backtrace_lines(exception, example)
+
+  cause = exception.cause
+  depth = 0
+  while cause && depth < 10
+    lines << ''
+    lines << "Caused by: #{cause.class}: #{cause.message}"
+    lines.concat(exception_backtrace_lines(cause, example))
+    cause = cause.cause
+    depth += 1
+  end
+
+  result = lines.join("\n")
+  # The exception presenter may choke on MultipleExceptionError, such as errors in before
+  # and after hooks, so we fall back to the legacy formatter
+  return legacy_format_exception_backtrace(exception) if result.strip.empty?
+
+  strip_ansi_codes(result)
+rescue StandardError
+  legacy_format_exception_backtrace(exception)
+end
+
+def exception_backtrace_lines(exception, example)
+  presenter = RSpec::Core::Formatters::ExceptionPresenter.new(exception, example)
+  Array(presenter.formatted_backtrace)
+rescue StandardError
+  Array(exception.backtrace)
+end
+
+def legacy_format_exception_message(exception)
+  case exception
+  when RSpec::Core::MultipleExceptionError
+    messages = exception.all_exceptions.map { |e| "#{e.class}: #{e.message}" }
+    strip_ansi_codes("#{exception.class}: #{messages.join(' | ')}")
+  else
+    strip_ansi_codes(exception.to_s)
+  end
+end
+
+# trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
+def legacy_format_exception_backtrace(exception)
+  case exception
+  when RSpec::Core::MultipleExceptionError
+    strip_ansi_codes(exception.all_exceptions.map do |e|
+      if e.backtrace && !e.backtrace.empty?
+        "#{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
+      else
+        "#{e.class}: #{e.message}"
+      end
+    end.join("\n\n"))
+  else
+    strip_ansi_codes(exception.backtrace&.join("\n") || '')
+  end
+end
+
+# TrunkAnalyticsListener is a class that is used to listen to the execution of the Example class
+# it generates and submits the final test reports
+class TrunkAnalyticsListener
+  MAX_TEXT_FIELD_SIZE = 8_000
+
+  def initialize
+    @testreport = $test_report
+  end
+
+  def example_finished(notification)
+    add_test_case(notification.example)
+  end
+
+  # trunk-ignore(rubocop/Metrics/MethodLength,rubocop/Metrics/AbcSize)
+  def close(_notification)
+    if $failure_encountered_and_quarantining_disabled
+      puts 'Note: Quarantining is disabled for this repo. Test failures were not quarantined.'.yellow
+    end
+    if $failure_encountered_and_quarantine_lookup_failed
+      puts 'Note: Failed to check quarantining status. Test failures were not quarantined.'.yellow
+    end
+
+    if ENV['TRUNK_LOCAL_UPLOAD_DIR']
+      saved = @testreport.try_save(ENV['TRUNK_LOCAL_UPLOAD_DIR'])
+      if saved
+        puts 'Local Flaky tests report generated'.green
+      else
+        puts 'Failed to generate local flaky tests report'.red
+      end
+    else
+      published = @testreport.publish
+      if published
+        puts 'Flaky tests report upload complete'.green
+      else
+        puts 'Failed to publish flaky tests report'.red
+      end
+    end
+  end
+
+  # trunk-ignore(rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength)
+  def add_test_case(example)
+    status, exception = status_and_exception(example)
+    failure_message = ''
+    backtrace = ''
+    if exception
+      failure_message = format_exception_message(exception, example).strip
+      backtrace = format_exception_backtrace(exception, example).strip
+    end
+    failure_message = failure_message[0...MAX_TEXT_FIELD_SIZE] if failure_message.length > MAX_TEXT_FIELD_SIZE
+    backtrace = backtrace[0...MAX_TEXT_FIELD_SIZE] if backtrace.length > MAX_TEXT_FIELD_SIZE
+    # TODO: should we use concatenated string or alias when auto-generated description?
+    name = example.full_description
+    file = escape(example.metadata[:file_path])
+    classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
+    line = example.metadata[:line_number]
+    started_at = example.execution_result.started_at.to_i
+    finished_at = example.execution_result.finished_at.to_i
+    id = example.generate_trunk_id
+
+    attempt_number = example.metadata[:retry_attempts] || example.metadata[:attempt_number] || 0
+    # set the status to failure, but mark it as quarantined
+    is_quarantined = example.metadata[:quarantined_exception] ? true : false
+    parent_name = example.example_group.metadata[:description]
+    parent_name = parent_name.empty? ? 'rspec' : parent_name
+    @testreport.add_test(id, name, classname, file, parent_name, line, status, attempt_number,
+                         started_at, finished_at, failure_message || '', backtrace || '', is_quarantined)
+  end
+
+  # Determine the Trunk status to report for an example, plus the exception (if
+  # any) whose message/backtrace should be recorded for it.
+  #
+  # `pending` examples need care because RSpec expresses their outcome as the
+  # inverse of the body's pass/fail. A pending example's contract is "this is
+  # expected to fail", so:
+  #   - body still fails  -> expectation met     -> RSpec status :pending, build green
+  #   - body now passes   -> expectation violated -> RSpec status :failed
+  #                          (PendingExampleFixedError), build red
+  # We report the outcome RSpec actually decided, which also matches the build
+  # result:
+  #   - skip / xit (never ran)     -> skipped
+  #   - pending, body still failing -> success (expectation met)
+  #   - pending, body now passing   -> failure (PendingExampleFixedError)
+  # RSpec's own build pass/fail behavior is left untouched (see #set_exception).
+  #
+  # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/MethodLength)
+  def status_and_exception(example)
+    result = example.execution_result
+    quarantined_exception = example.metadata[:quarantined_exception]
+
+    # A genuinely skipped example (`skip`/`xit`, or `skip` called from a hook or
+    # body) never runs, so it has no real pass/fail outcome. RSpec still reports
+    # it with status :pending, so detect skips explicitly -- and up front -- via
+    # metadata[:skip] before interpreting any pending pass/fail semantics below.
+    return [Status.new('skipped'), nil] if example.metadata[:skip]
+
+    case result.status
+    when :passed
+      # A quarantined failure is recorded as :passed by RSpec (see #set_exception),
+      # so report it as a failure but carry the original quarantined exception.
+      quarantined_exception ? [Status.new('failure'), quarantined_exception] : [Status.new('success'), nil]
+    when :failed
+      # Includes a pending example whose body unexpectedly passed: RSpec reports it
+      # as :failed with a PendingExampleFixedError (on example.exception) and breaks
+      # the build (the pending expectation was violated), so it is a failure here too.
+      [Status.new('failure'), example.exception || quarantined_exception]
+    when :pending
+      # Not a skip (handled above), so this is a pending example whose body ran and
+      # failed as expected: the pending expectation ("this should fail") was met, so
+      # RSpec keeps the build green -- report it as a success.
+      [Status.new('success'), nil]
+    else
+      [Status.new(result.status.to_s), example.exception || quarantined_exception]
+    end
+  end
+end
+
+RSpec.configure do |c|
+  next if trunk_disabled
+
+  c.before(:example) do
+    if $failure_encountered_and_quarantine_lookup_failed && quarantine_query_failure_exit?
+      skip('Quarantine lookup failed, skipping test run')
+    end
+  end
+
+  c.reporter.register_listener TrunkAnalyticsListener.new, :example_finished, :close
+end
+
+RSpec::Trunk.setup

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: rspec_trunk_flaky_tests
+version: !ruby/object:Gem::Version
+  version: 0.14.0.pre.beta.1
+platform: x86_64-linux-gnu
+authors:
+- Trunk Technologies, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2026-07-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Integrates RSpec with Trunk Flaky Tests to automatically upload test
+  results from your CI jobs. Enables accurate flaky test detection, quarantining,
+  and analytics.
+email: support@trunk.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/rspec_trunk_flaky_tests.rb
+- lib/rspec_trunk_flaky_tests/3.1/rspec_trunk_flaky_tests.so
+- lib/rspec_trunk_flaky_tests/3.2/rspec_trunk_flaky_tests.so
+- lib/rspec_trunk_flaky_tests/3.3/rspec_trunk_flaky_tests.so
+- lib/rspec_trunk_flaky_tests/3.4/rspec_trunk_flaky_tests.so
+- lib/rspec_trunk_flaky_tests/4.0/rspec_trunk_flaky_tests.so
+- lib/trunk_spec_helper.rb
+homepage: https://docs.trunk.io/flaky-tests/get-started/frameworks/rspec
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/trunk-io/analytics-cli/tree/main/rspec-trunk-flaky-tests
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 4.1.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.22
+requirements: []
+rubygems_version: 3.5.23
+signing_key: 
+specification_version: 4
+summary: RSpec plugin for Trunk Flaky Tests - automatically uploads test results to
+  detect and quarantine flaky tests
+test_files: []